thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into tostring-fix

Browse Source
This commit is contained in:
zhanglistar 2024-10-24 11:29:06 +08:00 committed by GitHub
commit 87e8a50a73
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
853 changed files with 15208 additions and 18094 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE/20_feature-request.md vendored
View File

@ -15,7 +15,7 @@ assignees: ''
**Use case**
> A clear and concise description of what is the intended usage scenario is.
> A clear and concise description of what the intended usage scenario is.
**Describe the solution you'd like**

2
CMakeLists.txt
View File

@ -545,7 +545,7 @@ endif()
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT ENABLE_FUZZING
AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
AND OMIT_HEAVY_DEBUG_SYMBOLS AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
else ()
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)

8
base/base/StringRef.h
View File

@ -369,11 +369,15 @@ namespace PackedZeroTraits
{
template <typename Second, template <typename, typename> class PackedPairNoInit>
inline bool check(const PackedPairNoInit<StringRef, Second> p)
{ return 0 == p.key.size; }
{
return 0 == p.key.size;
}
template <typename Second, template <typename, typename> class PackedPairNoInit>
inline void set(PackedPairNoInit<StringRef, Second> & p)
{ p.key.size = 0; }
{
p.key.size = 0;
}
}

5
base/glibc-compatibility/CMakeLists.txt
View File

@ -11,6 +11,9 @@ if (GLIBC_COMPATIBILITY)
if (ARCH_AARCH64)
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s)
set (musl_arch_include_dir musl/aarch64)
# Disable getauxval in aarch64. ARM glibc minimum requirement for the project is 2.18 and getauxval is present
# in 2.16. Having a custom one introduces issues with sanitizers
list (REMOVE_ITEM glibc_compatibility_sources musl/getauxval.c)
elseif (ARCH_AMD64)
list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s)
set (musl_arch_include_dir musl/x86_64)
@ -18,7 +21,7 @@ if (GLIBC_COMPATIBILITY)
message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
endif ()
if (SANITIZE STREQUAL thread)
if (SANITIZE STREQUAL thread AND ARCH_AMD64)
# Disable TSAN instrumentation that conflicts with re-exec due to high ASLR entropy using getauxval
# See longer comment in __auxv_init_procfs
# In the case of tsan we need to make sure getauxval is not instrumented as that would introduce tsan

2
base/poco/Foundation/include/Poco/Logger.h
View File

@ -952,6 +952,8 @@ private:
static std::pair<LoggerMapIterator, bool> add(Logger * pLogger);
static std::optional<LoggerMapIterator> find(const std::string & name);
static Logger * findRawPtr(const std::string & name);
void unsafeSetChannel(Channel * pChannel);
Channel* unsafeGetChannel() const;
Logger();
Logger(const Logger &);

21
base/poco/Foundation/src/Logger.cpp
View File

@ -61,6 +61,13 @@ Logger::~Logger()
void Logger::setChannel(Channel* pChannel)
{
std::lock_guard<std::mutex> lock(getLoggerMutex());
unsafeSetChannel(pChannel);
}
void Logger::unsafeSetChannel(Channel* pChannel)
{
if (_pChannel) _pChannel->release();
_pChannel = pChannel;
@ -69,6 +76,14 @@ void Logger::setChannel(Channel* pChannel)
Channel* Logger::getChannel() const
{
std::lock_guard<std::mutex> lock(getLoggerMutex());
return unsafeGetChannel();
}
Channel* Logger::unsafeGetChannel() const
{
return _pChannel;
}
@ -89,7 +104,7 @@ void Logger::setLevel(const std::string& level)
void Logger::setProperty(const std::string& name, const std::string& value)
{
if (name == "channel")
setChannel(LoggingRegistry::defaultRegistry().channelForName(value));
unsafeSetChannel(LoggingRegistry::defaultRegistry().channelForName(value));
else if (name == "level")
setLevel(value);
else
@ -160,7 +175,7 @@ void Logger::setChannel(const std::string& name, Channel* pChannel)
if (len == 0 ||
(it.first.compare(0, len, name) == 0 && (it.first.length() == len || it.first[len] == '.')))
{
it.second.logger->setChannel(pChannel);
it.second.logger->unsafeSetChannel(pChannel);
}
}
}
@ -393,7 +408,7 @@ std::pair<Logger::LoggerMapIterator, bool> Logger::unsafeGet(const std::string&
else
{
Logger& par = parent(name);
logger = new Logger(name, par.getChannel(), par.getLevel());
logger = new Logger(name, par.unsafeGetChannel(), par.getLevel());
}
return add(logger);

32
cmake/cpu_features.cmake
View File

@ -11,6 +11,38 @@ option (ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries n
if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
# Populate the ENABLE_ option flags. This is required for the build of some third-party dependencies, specifically snappy, which
# (somewhat weirdly) expects the relative SNAPPY_HAVE_ preprocessor variables to be populated, in addition to the microarchitecture
# feature flags being enabled in the compiler. This fixes the ARCH_NATIVE flag by automatically populating the ENABLE_ option flags
# according to the current CPU's capabilities, detected using clang.
if (ARCH_AMD64)
execute_process(
COMMAND sh -c "clang -E - -march=native -###"
INPUT_FILE /dev/null
OUTPUT_QUIET
ERROR_VARIABLE TEST_FEATURE_RESULT)
macro(TEST_AMD64_FEATURE TEST_FEATURE_RESULT feat flag)
if (${TEST_FEATURE_RESULT} MATCHES "\"\\+${feat}\"")
set(${flag} ON)
else ()
set(${flag} OFF)
endif ()
endmacro()
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} ssse3 ENABLE_SSSE3)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} sse4.1 ENABLE_SSE41)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} sse4.2 ENABLE_SSE42)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} vpclmulqdq ENABLE_PCLMULQDQ)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} popcnt ENABLE_POPCNT)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx ENABLE_AVX)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx2 ENABLE_AVX2)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx512f ENABLE_AVX512)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx512vbmi ENABLE_AVX512_VBMI)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} bmi ENABLE_BMI)
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} bmi2 ENABLE_BMI2)
endif ()
elseif (ARCH_AARCH64)
# ARM publishes almost every year a new revision of it's ISA [1]. Each version comes with new mandatory and optional features from
# which CPU vendors can pick and choose. This creates a lot of variability ... We provide two build "profiles", one for maximum

19
cmake/cxx.cmake
View File

@ -1,4 +1,21 @@
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
# Enable libcxx debug mode: https://releases.llvm.org/15.0.0/projects/libcxx/docs/DesignDocs/DebugMode.html
# The docs say the debug mode violates complexity guarantees, so do this only for Debug builds.
# set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_ENABLE_DEBUG_MODE=1")
# ^^ Crashes the database upon startup, needs investigation.
# Besides that, the implementation looks like a poor man's MSAN specific to libcxx. Since CI tests MSAN
# anyways, we can keep the debug mode disabled.
# Libcxx also provides extra assertions:
# --> https://releases.llvm.org/15.0.0/projects/libcxx/docs/UsingLibcxx.html#assertions-mode
# These look orthogonal to the debug mode but the debug mode enables them implicitly:
# --> https://github.com/llvm/llvm-project/blob/release/15.x/libcxx/include/__assert#L29
# They are cheap and straightforward, so enable them in debug builds:
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_ENABLE_ASSERTIONS=1")
# TODO Once we upgrade to LLVM 18+, reconsider all of the above as they introduced "hardening modes":
# https://libcxx.llvm.org/Hardening.html
endif ()
add_subdirectory(contrib/libcxxabi-cmake)
add_subdirectory(contrib/libcxx-cmake)

3
contrib/abseil-cpp-cmake/CMakeLists.txt
View File

@ -1,6 +1,9 @@
set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}")
# To avoid errors "'X' does not refer to a value" while using `offsetof` function.
set(CMAKE_CXX_STANDARD 17)
# This is a minimized version of the function definition in CMake/AbseilHelpers.cmake
#

3
contrib/google-protobuf-cmake/CMakeLists.txt
View File

@ -5,6 +5,9 @@ if(NOT ENABLE_PROTOBUF)
return()
endif()
# To avoid errors "'X' does not refer to a value" while using `offsetof` function.
set(CMAKE_CXX_STANDARD 17)
set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src")
if(OS_FREEBSD AND SANITIZE STREQUAL "address")
# ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found

2
contrib/grpc-cmake/CMakeLists.txt
View File

@ -6,6 +6,8 @@ if(NOT ENABLE_GRPC)
return()
endif()
set(CMAKE_CXX_STANDARD 17)
set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc")
set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc")

2
contrib/grpc-cmake/grpc.cmake
View File

@ -22,7 +22,7 @@
# limitations under the License.
# We want to use C++23, but GRPC is not ready
set (CMAKE_CXX_STANDARD 20)
set (CMAKE_CXX_STANDARD 17)
set(_gRPC_ZLIB_INCLUDE_DIR "")
set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib)

2
contrib/libhdfs3 vendored

@ -1 +1 @@
Subproject commit 0d04201c45359f0d0701fb1e8297d25eff7cfecf
Subproject commit de6f1e0750aa3670a603cbfeddf5df3de1097687

2
contrib/orc vendored

@ -1 +1 @@
Subproject commit bcc025c09828c556f54cfbdf83a66b9acae7d17f
Subproject commit 223e1957ff308f2aec7793884ffb0d7692d48484

2
contrib/usearch vendored

@ -1 +1 @@
Subproject commit d1d33eac94acd3b628e0b446c927ec3295ef63c7
Subproject commit 1706420acafbd83d852c512dcf343af0a4059e48

5
docker/test/integration/runner/Dockerfile
View File

@ -42,13 +42,12 @@ ENV TZ=Etc/UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
ENV DOCKER_CHANNEL stable
# Unpin the docker version after the release 24.0.3 is released
# https://github.com/moby/moby/issues/45770#issuecomment-1618255130
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
&& add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
&& apt-get update \
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
docker-ce='5:23.*' docker-compose-plugin='2.29.*' \
docker-ce="5:27.0.3*" \
docker-compose-plugin='2.29.*' \
&& rm -rf \
/var/lib/apt/lists/* \
/var/cache/debconf \

5
docker/test/integration/runner/misc/openldap/initialized.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -e
# workaround for https://github.com/bitnami/containers/issues/73310
touch /tmp/.openldap-initialized

14
docker/test/style/requirements.txt
View File

@ -1,4 +1,4 @@
aiohttp==3.9.5
aiohttp==3.10.2
aiosignal==1.3.1
astroid==3.1.0
async-timeout==4.0.3
@ -6,12 +6,12 @@ attrs==23.2.0
black==24.4.2
boto3==1.34.131
botocore==1.34.131
certifi==2024.6.2
certifi==2024.07.04
cffi==1.16.0
charset-normalizer==3.3.2
click==8.1.7
codespell==2.2.1
cryptography==42.0.8
cryptography==43.0.1
Deprecated==1.2.14
dill==0.3.8
flake8==4.0.1
@ -19,7 +19,6 @@ frozenlist==1.4.1
idna==3.7
isort==5.13.2
jmespath==1.0.1
jwt==1.3.1
mccabe==0.6.1
multidict==6.0.5
mypy==1.8.0
@ -27,13 +26,12 @@ mypy-extensions==1.0.0
packaging==24.1
pathspec==0.9.0
pip==24.1.1
pipdeptree==2.23.0
platformdirs==4.2.2
pycodestyle==2.8.0
pycparser==2.22
pyflakes==2.4.0
PyGithub==2.3.0
PyJWT==2.8.0
PyJWT==2.9.0
pylint==3.1.0
PyNaCl==1.5.0
python-dateutil==2.9.0.post0
@ -42,7 +40,7 @@ PyYAML==6.0.1
rapidfuzz==3.9.3
requests==2.32.3
s3transfer==0.10.1
setuptools==59.6.0
setuptools==70.0.0
six==1.16.0
thefuzz==0.22.1
tomli==2.0.1
@ -52,7 +50,7 @@ types-requests==2.32.0.20240622
typing_extensions==4.12.2
unidiff==0.7.5
urllib3==2.2.2
wheel==0.37.1
wheel==0.38.1
wrapt==1.16.0
yamllint==1.26.3
yarl==1.9.4

1
docs/README.md
View File

@ -196,7 +196,6 @@ When writing docs, you can use prepared templates. Copy the code of a template a
Templates:
- [Function](_description_templates/template-function.md)
- [Setting](_description_templates/template-setting.md)
- [Server Setting](_description_templates/template-server-setting.md)
- [Database or Table engine](_description_templates/template-engine.md)
- [System table](_description_templates/template-system-table.md)

27
docs/_description_templates/template-setting.md
View File

@ -1,27 +0,0 @@
## setting_name {#setting_name}
Description.
For the switch setting, use the typical phrase: “Enables or disables something ...”.
Possible values:
*For switcher setting:*
- 0 — Disabled.
- 1 — Enabled.
*For another setting (typical phrases):*
- Positive integer.
- 0 — Disabled or unlimited or something else.
Default value: `value`.
**Additional Info** (Optional)
The name of an additional section can be any, for example, **Usage**.
**See Also** (Optional)
- [link](#)

11
docs/_includes/install/deb.sh
View File

@ -1,11 +0,0 @@
sudo apt-get install -y apt-transport-https ca-certificates dirmngr
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
/etc/apt/sources.list.d/clickhouse.list
sudo apt-get update
sudo apt-get install -y clickhouse-server clickhouse-client
sudo service clickhouse-server start
clickhouse-client # or "clickhouse-client --password" if you've set up a password.

6
docs/_includes/install/rpm.sh
View File

@ -1,6 +0,0 @@
sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo
sudo yum install -y clickhouse-server clickhouse-client
sudo /etc/init.d/clickhouse-server start
clickhouse-client # or "clickhouse-client --password" if you set up a password.

32
docs/_includes/install/tgz.sh
View File

@ -1,32 +0,0 @@
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
export LATEST_VERSION
case $(uname -m) in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "Unknown architecture $(uname -m)"; exit 1 ;;
esac
for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
do
curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
|| curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
done
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" configure
sudo /etc/init.d/clickhouse-server start
tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
|| tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"

7
docs/_includes/install/universal.sh
View File

@ -14,7 +14,12 @@ then
HAS_SSE42=$(grep sse4_2 /proc/cpuinfo)
if [ "${HAS_SSE42}" ]
then
DIR="amd64"
if ldd --version 2>&1 | grep -q musl
then
DIR="amd64musl"
else
DIR="amd64"
fi
else
DIR="amd64compat"
fi

3
docs/en/development/build-osx.md
View File

@ -47,6 +47,9 @@ git clone --recursive git@github.com:ClickHouse/ClickHouse.git
# ...alternatively, you can use https://github.com/ClickHouse/ClickHouse.git as the repo URL.
```
Apple uses a case-insensitive file system by default. While this usually does not affect compilation (especially scratch makes will work), it can confuse file operations like `git mv`.
For serious development on macOS, make sure that the source code is stored on a case-sensitive disk volume, e.g. see [these instructions](https://brianboyko.medium.com/a-case-sensitive-src-folder-for-mac-programmers-176cc82a3830).
## Build ClickHouse {#build-clickhouse}
To build using Homebrew's vanilla Clang compiler (the only **recommended** way):

312
docs/en/engines/database-engines/materialized-mysql.md
View File

@ -7,315 +7,5 @@ sidebar_position: 70
# [experimental] MaterializedMySQL
:::note
This database engine is experimental. To use it, set `allow_experimental_database_materialized_mysql` to 1 in your configuration files or by using the `SET` command:
```sql
SET allow_experimental_database_materialized_mysql=1
```
This database engine is obsolete and cannot be used.
:::
Creates a ClickHouse database with all the tables existing in MySQL, and all the data in those tables. The ClickHouse server works as MySQL replica. It reads `binlog` and performs DDL and DML queries.
## Creating a Database {#creating-a-database}
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
[TABLE OVERRIDE table1 (...), TABLE OVERRIDE table2 (...)]
```
**Engine Parameters**
- `host:port` — MySQL server endpoint.
- `database` — MySQL database name.
- `user` — MySQL user.
- `password` — User password.
## Engine Settings
### max_rows_in_buffer
`max_rows_in_buffer` — Maximum number of rows that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `65 505`.
### max_bytes_in_buffer
`max_bytes_in_buffer` — Maximum number of bytes that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `1 048 576`.
### max_flush_data_time
`max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`.
### max_wait_time_when_mysql_unavailable
`max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`.
### allows_query_when_mysql_lost
`allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`).
### allow_startup_database_without_connection_to_mysql
`allow_startup_database_without_connection_to_mysql` — Allow to create and attach database without available connection to MySQL. Default: `0` (`false`).
### materialized_mysql_tables_list
`materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated.
```sql
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
SETTINGS
allows_query_when_mysql_lost=true,
max_wait_time_when_mysql_unavailable=10000;
```
## Settings on MySQL-server Side
For the correct work of `MaterializedMySQL`, there are few mandatory `MySQL`-side configuration settings that must be set:
### default_authentication_plugin
`default_authentication_plugin = mysql_native_password` since `MaterializedMySQL` can only authorize with this method.
### gtid_mode
`gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializedMySQL` replication.
:::note
While turning on `gtid_mode` you should also specify `enforce_gtid_consistency = on`.
:::
## Virtual Columns {#virtual-columns}
When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md) tables are used with virtual `_sign` and `_version` columns.
### \_version
`_version` — Transaction counter. Type [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
### \_sign
`_sign` — Deletion mark. Type [Int8](/docs/en/sql-reference/data-types/int-uint.md). Possible values:
- `1` — Row is not deleted,
- `-1` — Row is deleted.
## Data Types Support {#data_types-support}
| MySQL | ClickHouse |
|-------------------------|--------------------------------------------------------------|
| TINY | [Int8](/docs/en/sql-reference/data-types/int-uint.md) |
| SHORT | [Int16](/docs/en/sql-reference/data-types/int-uint.md) |
| INT24 | [Int32](/docs/en/sql-reference/data-types/int-uint.md) |
| LONG | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) |
| LONGLONG | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) |
| FLOAT | [Float32](/docs/en/sql-reference/data-types/float.md) |
| DOUBLE | [Float64](/docs/en/sql-reference/data-types/float.md) |
| DECIMAL, NEWDECIMAL | [Decimal](/docs/en/sql-reference/data-types/decimal.md) |
| DATE, NEWDATE | [Date](/docs/en/sql-reference/data-types/date.md) |
| DATETIME, TIMESTAMP | [DateTime](/docs/en/sql-reference/data-types/datetime.md) |
| DATETIME2, TIMESTAMP2 | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) |
| YEAR | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) |
| TIME | [Int64](/docs/en/sql-reference/data-types/int-uint.md) |
| ENUM | [Enum](/docs/en/sql-reference/data-types/enum.md) |
| STRING | [String](/docs/en/sql-reference/data-types/string.md) |
| VARCHAR, VAR_STRING | [String](/docs/en/sql-reference/data-types/string.md) |
| BLOB | [String](/docs/en/sql-reference/data-types/string.md) |
| GEOMETRY | [String](/docs/en/sql-reference/data-types/string.md) |
| BINARY | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| BIT | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) |
| SET | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) |
[Nullable](/docs/en/sql-reference/data-types/nullable.md) is supported.
The data of TIME type in MySQL is converted to microseconds in ClickHouse.
Other types are not supported. If MySQL table contains a column of such type, ClickHouse throws an exception and stops replication.
## Specifics and Recommendations {#specifics-and-recommendations}
### Compatibility Restrictions {#compatibility-restrictions}
Apart of the data types limitations there are few restrictions comparing to `MySQL` databases, that should be resolved before replication will be possible:
- Each table in `MySQL` should contain `PRIMARY KEY`.
- Replication for tables, those are containing rows with `ENUM` field values out of range (specified in `ENUM` signature) will not work.
### DDL Queries {#ddl-queries}
MySQL DDL queries are converted into the corresponding ClickHouse DDL queries ([ALTER](/docs/en/sql-reference/statements/alter/index.md), [CREATE](/docs/en/sql-reference/statements/create/index.md), [DROP](/docs/en/sql-reference/statements/drop.md), [RENAME](/docs/en/sql-reference/statements/rename.md)). If ClickHouse cannot parse some DDL query, the query is ignored.
### Data Replication {#data-replication}
`MaterializedMySQL` does not support direct `INSERT`, `DELETE` and `UPDATE` queries. However, they are supported in terms of data replication:
- MySQL `INSERT` query is converted into `INSERT` with `_sign=1`.
- MySQL `DELETE` query is converted into `INSERT` with `_sign=-1`.
- MySQL `UPDATE` query is converted into `INSERT` with `_sign=-1` and `INSERT` with `_sign=1` if the primary key has been changed, or
`INSERT` with `_sign=1` if not.
### Selecting from MaterializedMySQL Tables {#select}
`SELECT` query from `MaterializedMySQL` tables has some specifics:
- If `_version` is not specified in the `SELECT` query, the
[FINAL](/docs/en/sql-reference/statements/select/from.md/#select-from-final) modifier is used, so only rows with
`MAX(_version)` are returned for each primary key value.
- If `_sign` is not specified in the `SELECT` query, `WHERE _sign=1` is used by default. So the deleted rows are not
included into the result set.
- The result includes columns comments in case they exist in MySQL database tables.
### Index Conversion {#index-conversion}
MySQL `PRIMARY KEY` and `INDEX` clauses are converted into `ORDER BY` tuples in ClickHouse tables.
ClickHouse has only one physical order, which is determined by `ORDER BY` clause. To create a new physical order, use
[materialized views](/docs/en/sql-reference/statements/create/view.md/#materialized).
**Notes**
- Rows with `_sign=-1` are not deleted physically from the tables.
- Cascade `UPDATE/DELETE` queries are not supported by the `MaterializedMySQL` engine, as they are not visible in the
MySQL binlog.
- Replication can be easily broken.
- Manual operations on database and tables are forbidden.
- `MaterializedMySQL` is affected by the [optimize_on_insert](/docs/en/operations/settings/settings.md/#optimize-on-insert)
setting. Data is merged in the corresponding table in the `MaterializedMySQL` database when a table in the MySQL
server changes.
### Table Overrides {#table-overrides}
Table overrides can be used to customize the ClickHouse DDL queries, allowing you to make schema optimizations for your
application. This is especially useful for controlling partitioning, which is important for the overall performance of
MaterializedMySQL.
These are the schema conversion manipulations you can do with table overrides for MaterializedMySQL:
* Modify column type. Must be compatible with the original type, or replication will fail. For example,
you can modify a UInt32 column to UInt64, but you can not modify a String column to Array(String).
* Modify [column TTL](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-column-ttl).
* Modify [column compression codec](/docs/en/sql-reference/statements/create/table.md/#codecs).
* Add [ALIAS columns](/docs/en/sql-reference/statements/create/table.md/#alias).
* Add [skipping indexes](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#table_engine-mergetree-data_skipping-indexes). Note that you need to enable `use_skip_indexes_if_final` setting to make them work (MaterializedMySQL is using `SELECT ... FINAL` by default)
* Add [projections](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#projections). Note that projection optimizations are
disabled when using `SELECT ... FINAL` (which MaterializedMySQL does by default), so their utility is limited here.
`INDEX ... TYPE hypothesis` as [described in the v21.12 blog post]](https://clickhouse.com/blog/en/2021/clickhouse-v21.12-released/)
may be more useful in this case.
* Modify [PARTITION BY](/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key/)
* Modify [ORDER BY](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
* Modify [PRIMARY KEY](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
* Add [SAMPLE BY](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
* Add [table TTL](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
```sql
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
[SETTINGS ...]
[TABLE OVERRIDE table_name (
[COLUMNS (
[col_name [datatype] [ALIAS expr] [CODEC(...)] [TTL expr], ...]
[INDEX index_name expr TYPE indextype[(...)] GRANULARITY val, ...]
[PROJECTION projection_name (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
)]
[ORDER BY expr]
[PRIMARY KEY expr]
[PARTITION BY expr]
[SAMPLE BY expr]
[TTL expr]
), ...]
```
Example:
```sql
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
TABLE OVERRIDE table1 (
COLUMNS (
userid UUID,
category LowCardinality(String),
timestamp DateTime CODEC(Delta, Default)
)
PARTITION BY toYear(timestamp)
),
TABLE OVERRIDE table2 (
COLUMNS (
client_ip String TTL created + INTERVAL 72 HOUR
)
SAMPLE BY ip_hash
)
```
The `COLUMNS` list is sparse; existing columns are modified as specified, extra ALIAS columns are added. It is not
possible to add ordinary or MATERIALIZED columns. Modified columns with a different type must be assignable from the
original type. There is currently no validation of this or similar issues when the `CREATE DATABASE` query executes, so
extra care needs to be taken.
You may specify overrides for tables that do not exist yet.
:::important
It is easy to break replication with table overrides if not used with care. For example:
* If an ALIAS column is added with a table override, and a column with the same name is later added to the source
MySQL table, the converted ALTER TABLE query in ClickHouse will fail and replication stops.
* It is currently possible to add overrides that reference nullable columns where not-nullable are required, such as in
`ORDER BY` or `PARTITION BY`. This will cause CREATE TABLE queries that will fail, also causing replication to stop.
:::
## Examples of Use {#examples-of-use}
Queries in MySQL:
``` sql
mysql> CREATE DATABASE db;
mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
mysql> DELETE FROM db.test WHERE a=1;
mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
mysql> UPDATE db.test SET c='Wow!', b=222;
mysql> SELECT * FROM test;
```
```text
┌─a─┬───b─┬─c────┐
│ 2 │ 222 │ Wow! │
└───┴─────┴──────┘
```
Database in ClickHouse, exchanging data with the MySQL server:
The database and the table created:
``` sql
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
SHOW TABLES FROM mysql;
```
``` text
┌─name─┐
│ test │
└──────┘
```
After inserting data:
``` sql
SELECT * FROM mysql.test;
```
``` text
┌─a─┬──b─┐
│ 1 │ 11 │
│ 2 │ 22 │
└───┴────┘
```
After deleting data, adding the column and updating:
``` sql
SELECT * FROM mysql.test;
```
``` text
┌─a─┬───b─┬─c────┐
│ 2 │ 222 │ Wow! │
└───┴─────┴──────┘
```

1
docs/en/engines/table-engines/integrations/azure-queue.md
View File

@ -36,6 +36,7 @@ SETTINGS
## Settings {#settings}
The set of supported settings is the same as for `S3Queue` table engine, but without `s3queue_` prefix. See [full list of settings settings](../../../engines/table-engines/integrations/s3queue.md#settings).
To get a list of settings, configured for the table, use `system.s3_queue_settings` table. Available from `24.10`.
## Description {#description}

27
docs/en/engines/table-engines/integrations/azureBlobStorage.md
View File

@ -63,7 +63,34 @@ Currently there are 3 ways to authenticate:
- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url.
- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication.
### Data cache {#data-cache}
`Azure` table engine supports data caching on local disk.
See filesystem cache configuration options and usage in this [section](/docs/en/operations/storing-data.md/#using-local-cache).
Caching is made depending on the path and ETag of the storage object, so clickhouse will not read a stale cache version.
To enable caching use a setting `filesystem_cache_name = '<name>'` and `enable_filesystem_cache = 1`.
```sql
SELECT *
FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', 'test_container', 'test_table', 'CSV')
SETTINGS filesystem_cache_name = 'cache_for_azure', enable_filesystem_cache = 1;
```
1. add the following section to clickhouse configuration file:
``` xml
<clickhouse>
<filesystem_caches>
<cache_for_azure>
<path>path to cache directory</path>
<max_size>10Gi</max_size>
</cache_for_azure>
</filesystem_caches>
</clickhouse>
```
2. reuse cache configuration (and therefore cache storage) from clickhouse `storage_configuration` section, [described here](/docs/en/operations/storing-data.md/#using-local-cache)
## See also

4
docs/en/engines/table-engines/integrations/deltalake.md
View File

@ -48,6 +48,10 @@ Using named collections:
CREATE TABLE deltalake ENGINE=DeltaLake(deltalake_conf, filename = 'test_table')
```
### Data cache {#data-cache}
`Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache).
## See also
- [deltaLake table function](../../../sql-reference/table-functions/deltalake.md)

11
docs/en/engines/table-engines/integrations/iceberg.md
View File

@ -6,7 +6,7 @@ sidebar_label: Iceberg
# Iceberg Table Engine
This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables.
This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS and locally stored tables.
## Create Table
@ -19,13 +19,16 @@ CREATE TABLE iceberg_table_s3
CREATE TABLE iceberg_table_azure
ENGINE = IcebergAzure(connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression])
CREATE TABLE iceberg_table_hdfs
ENGINE = IcebergHDFS(path_to_table, [,format] [,compression_method])
CREATE TABLE iceberg_table_local
ENGINE = IcebergLocal(path_to_table, [,format] [,compression_method])
```
**Engine arguments**
Description of the arguments coincides with description of arguments in engines `S3`, `AzureBlobStorage` and `File` correspondingly.
Description of the arguments coincides with description of arguments in engines `S3`, `AzureBlobStorage`, `HDFS` and `File` correspondingly.
`format` stands for the format of data files in the Iceberg table.
Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md)
@ -60,6 +63,10 @@ CREATE TABLE iceberg_table ENGINE=IcebergS3(iceberg_conf, filename = 'test_table
Table engine `Iceberg` is an alias to `IcebergS3` now.
### Data cache {#data-cache}
`Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache).
## See also
- [iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md)

2
docs/en/engines/table-engines/integrations/mongodb.md
View File

@ -9,7 +9,7 @@ sidebar_label: MongoDB
MongoDB engine is read-only table engine which allows to read data from remote [MongoDB](https://www.mongodb.com/) collection.
Only MongoDB v3.6+ servers are supported.
[Seed list(`mongodb**+srv**`)](https://www.mongodb.com/docs/manual/reference/glossary/#std-term-seed-list) is not yet supported.
[Seed list(`mongodb+srv`)](https://www.mongodb.com/docs/manual/reference/glossary/#std-term-seed-list) is not yet supported.
:::note
If you're facing troubles, please report the issue, and try to use [the legacy implementation](../../../operations/server-configuration-parameters/settings.md#use_legacy_mongodb_integration).

4
docs/en/engines/table-engines/integrations/mysql.md
View File

@ -4,12 +4,8 @@ sidebar_position: 138
sidebar_label: MySQL
---
import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge';
# MySQL Table Engine
<CloudAvailableBadge />
The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.
## Creating a Table {#creating-a-table}

32
docs/en/engines/table-engines/integrations/s3.md
View File

@ -26,6 +26,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
│ two │ 2 │
└──────┴───────┘
```
## Create Table {#creating-a-table}
``` sql
@ -43,6 +44,37 @@ CREATE TABLE s3_engine_table (name String, value UInt32)
- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3).
- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension.
### Data cache {#data-cache}
`S3` table engine supports data caching on local disk.
See filesystem cache configuration options and usage in this [section](/docs/en/operations/storing-data.md/#using-local-cache).
Caching is made depending on the path and ETag of the storage object, so clickhouse will not read a stale cache version.
To enable caching use a setting `filesystem_cache_name = '<name>'` and `enable_filesystem_cache = 1`.
```sql
SELECT *
FROM s3('http://minio:10000/clickhouse//test_3.csv', 'minioadmin', 'minioadminpassword', 'CSV')
SETTINGS filesystem_cache_name = 'cache_for_s3', enable_filesystem_cache = 1;
```
There are two ways to define cache in configuration file.
1. add the following section to clickhouse configuration file:
``` xml
<clickhouse>
<filesystem_caches>
<cache_for_s3>
<path>path to cache directory</path>
<max_size>10Gi</max_size>
</cache_for_s3>
</filesystem_caches>
</clickhouse>
```
2. reuse cache configuration (and therefore cache storage) from clickhouse `storage_configuration` section, [described here](/docs/en/operations/storing-data.md/#using-local-cache)
### PARTITION BY
`PARTITION BY` — Optional. In most cases you don't need a partition key, and if it is needed you generally don't need a partition key more granular than by month. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead, make client identifier or name the first column in the ORDER BY expression).

2
docs/en/engines/table-engines/integrations/s3queue.md
View File

@ -69,6 +69,8 @@ SETTINGS
## Settings {#settings}
To get a list of settings, configured for the table, use `system.s3_queue_settings` table. Available from `24.10`.
### mode {#mode}
Possible values:

22
docs/en/engines/table-engines/mergetree-family/annindexes.md
View File

@ -43,7 +43,7 @@ CREATE TABLE table
(
id Int64,
vectors Array(Float32),
INDEX index_name vectors TYPE vector_similarity(method, distance_function[, quantization, connectivity, expansion_add, expansion_search]) [GRANULARITY N]
INDEX index_name vectors TYPE vector_similarity(method, distance_function[, quantization, hnsw_max_connections_per_layer, hnsw_candidate_list_size_for_construction]) [GRANULARITY N]
)
ENGINE = MergeTree
ORDER BY id;
@ -55,11 +55,13 @@ Parameters:
line between two points in Euclidean space), or `cosineDistance` (the [cosine
distance](https://en.wikipedia.org/wiki/Cosine_similarity#Cosine_distance)- the angle between two non-zero vectors).
- `quantization`: either `f64`, `f32`, `f16`, `bf16`, or `i8` for storing the vector with reduced precision (optional, default: `bf16`)
- `m`: the number of neighbors per graph node (optional, default: 16)
- `ef_construction`: (optional, default: 128)
- `ef_search`: (optional, default: 64)
- `hnsw_max_connections_per_layer`: the number of neighbors per HNSW graph node, also known as `M` in the [HNSW
paper](https://doi.org/10.1109/TPAMI.2018.2889473) (optional, default: 32)
- `hnsw_candidate_list_size_for_construction`: the size of the dynamic candidate list when constructing the HNSW graph, also known as
`ef_construction` in the original [HNSW paper](https://doi.org/10.1109/TPAMI.2018.2889473) (optional, default: 128)
Value 0 for parameters `m`, `ef_construction`, and `ef_search` refers to the default value.
Values 0 for parameters `hnsw_max_connections_per_layer` and `hnsw_candidate_list_size_for_construction` means using the default values of
these parameters.
Example:
@ -115,6 +117,11 @@ ANN indexes are built during column insertion and merge. As a result, `INSERT` a
tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write
requests.
:::tip
To reduce the cost of building vector similarity indexes, consider setting `materialize_skip_indexes_on_insert` which disables the
construction of skipping indexes on newly inserted parts. Search would fall back to exact search but as inserted parts are typically small
compared to the total table size, the performance impact of that would be negligible.
ANN indexes support this type of query:
``` sql
@ -124,6 +131,7 @@ FROM table
WHERE ... -- WHERE clause is optional
ORDER BY Distance(vectors, reference_vector)
LIMIT N
SETTINGS enable_analyzer = 0; -- Temporary limitation, will be lifted
```
:::tip
@ -135,6 +143,10 @@ clickhouse-client --param_vec='hello' --query="SELECT * FROM table WHERE L2Dista
```
:::
To search using a different value of HNSW parameter `hnsw_candidate_list_size_for_search` (default: 256), also known as `ef_search` in the
original [HNSW paper](https://doi.org/10.1109/TPAMI.2018.2889473), run the `SELECT` query with `SETTINGS hnsw_candidate_list_size_for_search
= <value>`.
**Restrictions**: Approximate algorithms used to determine the nearest neighbors require a limit, hence queries without `LIMIT` clause
cannot utilize ANN indexes. Also, ANN indexes are only used if the query has a `LIMIT` value smaller than setting
`max_limit_for_ann_queries` (default: 1 million rows). This is a safeguard to prevent large memory allocations by external libraries for

711
docs/en/getting-started/example-datasets/tpch.md
View File

File diff suppressed because it is too large Load Diff

55
docs/en/interfaces/formats.md
View File

@ -2054,35 +2054,40 @@ ClickHouse Avro format supports reading and writing [Avro data files](https://av
The table below shows supported data types and how they match ClickHouse [data types](/docs/en/sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries.
| Avro data type `INSERT` | ClickHouse data type | Avro data type `SELECT` |
|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|-------------------------------|
| `boolean`, `int`, `long`, `float`, `double` | [Int(8\16\32)](/docs/en/sql-reference/data-types/int-uint.md), [UInt(8\16\32)](/docs/en/sql-reference/data-types/int-uint.md) | `int` |
| `boolean`, `int`, `long`, `float`, `double` | [Int64](/docs/en/sql-reference/data-types/int-uint.md), [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `long` |
| `boolean`, `int`, `long`, `float`, `double` | [Float32](/docs/en/sql-reference/data-types/float.md) | `float` |
| `boolean`, `int`, `long`, `float`, `double` | [Float64](/docs/en/sql-reference/data-types/float.md) | `double` |
| `bytes`, `string`, `fixed`, `enum` | [String](/docs/en/sql-reference/data-types/string.md) | `bytes` or `string` \* |
| `bytes`, `string`, `fixed` | [FixedString(N)](/docs/en/sql-reference/data-types/fixedstring.md) | `fixed(N)` |
| `enum` | [Enum(8\16)](/docs/en/sql-reference/data-types/enum.md) | `enum` |
| `array(T)` | [Array(T)](/docs/en/sql-reference/data-types/array.md) | `array(T)` |
| `map(V, K)` | [Map(V, K)](/docs/en/sql-reference/data-types/map.md) | `map(string, K)` |
| `union(null, T)`, `union(T, null)` | [Nullable(T)](/docs/en/sql-reference/data-types/date.md) | `union(null, T)` |
| `null` | [Nullable(Nothing)](/docs/en/sql-reference/data-types/special-data-types/nothing.md) | `null` |
| `int (date)` \** | [Date](/docs/en/sql-reference/data-types/date.md), [Date32](docs/en/sql-reference/data-types/date32.md) | `int (date)` \** |
| `long (timestamp-millis)` \** | [DateTime64(3)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \** |
| `long (timestamp-micros)` \** | [DateTime64(6)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \** |
| `bytes (decimal)` \** | [DateTime64(N)](/docs/en/sql-reference/data-types/datetime.md) | `bytes (decimal)` \** |
| `int` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `int` |
| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `fixed(16)` |
| `bytes (decimal)` \** | [Decimal(P, S)](/docs/en/sql-reference/data-types/decimal.md) | `bytes (decimal)` \** |
| `string (uuid)` \** | [UUID](/docs/en/sql-reference/data-types/uuid.md) | `string (uuid)` \** |
| `fixed(16)` | [Int128/UInt128](/docs/en/sql-reference/data-types/int-uint.md) | `fixed(16)` |
| `fixed(32)` | [Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `fixed(32)` |
| `record` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `record` |
| Avro data type `INSERT` | ClickHouse data type | Avro data type `SELECT` |
|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|---------------------------------|
| `boolean`, `int`, `long`, `float`, `double` | [Int(8\16\32)](/docs/en/sql-reference/data-types/int-uint.md), [UInt(8\16\32)](/docs/en/sql-reference/data-types/int-uint.md) | `int` |
| `boolean`, `int`, `long`, `float`, `double` | [Int64](/docs/en/sql-reference/data-types/int-uint.md), [UInt64](/docs/en/sql-reference/data-types/int-uint.md) | `long` |
| `boolean`, `int`, `long`, `float`, `double` | [Float32](/docs/en/sql-reference/data-types/float.md) | `float` |
| `boolean`, `int`, `long`, `float`, `double` | [Float64](/docs/en/sql-reference/data-types/float.md) | `double` |
| `bytes`, `string`, `fixed`, `enum` | [String](/docs/en/sql-reference/data-types/string.md) | `bytes` or `string` \* |
| `bytes`, `string`, `fixed` | [FixedString(N)](/docs/en/sql-reference/data-types/fixedstring.md) | `fixed(N)` |
| `enum` | [Enum(8\16)](/docs/en/sql-reference/data-types/enum.md) | `enum` |
| `array(T)` | [Array(T)](/docs/en/sql-reference/data-types/array.md) | `array(T)` |
| `map(V, K)` | [Map(V, K)](/docs/en/sql-reference/data-types/map.md) | `map(string, K)` |
| `union(null, T)`, `union(T, null)` | [Nullable(T)](/docs/en/sql-reference/data-types/date.md) | `union(null, T)` |
| `union(T1, T2, …)` \** | [Variant(T1, T2, …)](/docs/en/sql-reference/data-types/variant.md) | `union(T1, T2, …)` \** |
| `null` | [Nullable(Nothing)](/docs/en/sql-reference/data-types/special-data-types/nothing.md) | `null` |
| `int (date)` \**\* | [Date](/docs/en/sql-reference/data-types/date.md), [Date32](docs/en/sql-reference/data-types/date32.md) | `int (date)` \**\* |
| `long (timestamp-millis)` \**\* | [DateTime64(3)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \**\* |
| `long (timestamp-micros)` \**\* | [DateTime64(6)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \**\* |
| `bytes (decimal)` \**\* | [DateTime64(N)](/docs/en/sql-reference/data-types/datetime.md) | `bytes (decimal)` \**\* |
| `int` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `int` |
| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `fixed(16)` |
| `bytes (decimal)` \**\* | [Decimal(P, S)](/docs/en/sql-reference/data-types/decimal.md) | `bytes (decimal)` \**\* |
| `string (uuid)` \**\* | [UUID](/docs/en/sql-reference/data-types/uuid.md) | `string (uuid)` \**\* |
| `fixed(16)` | [Int128/UInt128](/docs/en/sql-reference/data-types/int-uint.md) | `fixed(16)` |
| `fixed(32)` | [Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `fixed(32)` |
| `record` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `record` |
\* `bytes` is default, controlled by [output_format_avro_string_column_pattern](/docs/en/operations/settings/settings-formats.md/#output_format_avro_string_column_pattern)
\** [Avro logical types](https://avro.apache.org/docs/current/spec.html#Logical+Types)
\** [Variant type](/docs/en/sql-reference/data-types/variant) implicitly accepts `null` as a field value, so for example the Avro `union(T1, T2, null)` will be converted to `Variant(T1, T2)`.
As a result, when producing Avro from ClickHouse, we have to always include the `null` type to the Avro `union` type set as we don't know if any value is actually `null` during the schema inference.
\**\* [Avro logical types](https://avro.apache.org/docs/current/spec.html#Logical+Types)
Unsupported Avro logical data types: `time-millis`, `time-micros`, `duration`

4
docs/en/operations/opentelemetry.md
View File

@ -31,6 +31,10 @@ The table must be enabled in the server configuration, see the `opentelemetry_sp
The tags or attributes are saved as two parallel arrays, containing the keys and values. Use [ARRAY JOIN](../sql-reference/statements/select/array-join.md) to work with them.
## Log-query-settings
Setting [log_query_settings](settings/settings.md) allows log changes to query settings during query execution. When enabled, any modifications made to query settings will be recorded in the OpenTelemetry span log. This feature is particularly useful in production environments for tracking configuration changes that may affect query performance.
## Integration with monitoring systems
At the moment, there is no ready tool that can export the tracing data from ClickHouse to a monitoring system.

24
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -1488,6 +1488,8 @@ Keys:
- `formatting` – Log format for console output. Currently, only `json` is supported).
- `use_syslog` - Also forward log output to syslog.
- `syslog_level` - Log level for logging to syslog.
- `message_regexp` - Only log messages that match this regular expression. Defaults to `""`, indicating no filtering.
- `message_regexp_negative` - Only log messages that don't match this regular expression. Defaults to `""`, indicating no filtering.
**Log format specifiers**
@ -1576,6 +1578,28 @@ The log level of individual log names can be overridden. For example, to mute al
</logger>
```
**Regular Expression Filtering**
The messages logged can be filtered using regular expressions using `message_regexp` and `message_regexp_negative`. This can be done on a per-level basis or globally. If both a global and logger-specific pattern is specified, the global pattern is overridden (ignored) and only the logger-specific pattern applies. The positive and negative patterns are considered independently for this situation. Note: Using this feature may cause a slight slowdown in performance.
```xml
<logger>
<level>trace</level>
<!-- Global: Don't log Trace messages -->
<message_regexp_negative>.*Trace.*</message_regexp_negative>
<message_regexps>
<logger>
<!-- For the executeQuery logger, only log if message has "Read", but not "from" -->
<name>executeQuery</name>
<message_regexp>.*Read.*</message_regexp>
<message_regexp_negative>.*from.*</message_regexp_negative>
</logger>
</message_regexps>
</logger>
```
### syslog
To write log messages additionally to syslog:

1
docs/en/operations/settings/index.md
View File

@ -2,7 +2,6 @@
title: "Settings Overview"
sidebar_position: 1
slug: /en/operations/settings/
pagination_next: en/operations/settings/settings
---
# Settings Overview

4
docs/en/operations/settings/merge-tree-settings.md
View File

@ -49,7 +49,7 @@ Default value: 8192.
Maximum size of data granules in bytes.
Default value: 10Mb.
Default value: 10485760 (ca. 10 MiB).
To restrict the granule size only by number of rows, set to 0 (not recommended).
@ -1079,6 +1079,8 @@ Possible values:
Default value: 0 bytes.
Note that if both `min_free_disk_bytes_to_perform_insert` and `min_free_disk_ratio_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.
## min_free_disk_ratio_to_perform_insert
The minimum free to total disk space ratio to perform an `INSERT`. Must be a floating point value between 0 and 1. Note that this setting:

176
docs/en/operations/settings/mysql-binlog-client.md
View File

@ -1,176 +0,0 @@
# The MySQL Binlog Client
The MySQL Binlog Client provides a mechanism in ClickHouse to share the binlog from a MySQL instance among multiple [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) databases. This avoids consuming unnecessary bandwidth and CPU when replicating more than one schema/database.
The implementation is resilient against crashes and disk issues. The executed GTID sets of the binlog itself and the consuming databases have persisted only after the data they describe has been safely persisted as well. The implementation also tolerates re-doing aborted operations (at-least-once delivery).
# Settings
## use_binlog_client
Forces to reuse existing MySQL binlog connection or creates new one if does not exist. The connection is defined by `user:pass@host:port`.
Default value: 0
**Example**
```sql
-- create MaterializedMySQL databases that read the events from the binlog client
CREATE DATABASE db1 ENGINE = MaterializedMySQL('host:port', 'db1', 'user', 'password') SETTINGS use_binlog_client=1
CREATE DATABASE db2 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1
CREATE DATABASE db3 ENGINE = MaterializedMySQL('host:port', 'db3', 'user2', 'password2') SETTINGS use_binlog_client=1
```
Databases `db1` and `db2` will use the same binlog connection, since they use the same `user:pass@host:port`. Database `db3` will use separate binlog connection.
## max_bytes_in_binlog_queue
Defines the limit of bytes in the events binlog queue. If bytes in the queue increases this limit, it will stop reading new events from MySQL until the space for new events will be freed. This introduces the memory limits. Very high value could consume all available memory. Very low value could make the databases to wait for new events.
Default value: 67108864
**Example**
```sql
CREATE DATABASE db1 ENGINE = MaterializedMySQL('host:port', 'db1', 'user', 'password') SETTINGS use_binlog_client=1, max_bytes_in_binlog_queue=33554432
CREATE DATABASE db2 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1
```
If database `db1` is unable to consume binlog events fast enough and the size of the events queue exceeds `33554432` bytes, reading of new events from MySQL is postponed until `db1`
consumes the events and releases some space.
NOTE: This will impact to `db2`, and it will be waiting for new events too, since they share the same connection.
## max_milliseconds_to_wait_in_binlog_queue
Defines the max milliseconds to wait when `max_bytes_in_binlog_queue` exceeded. After that it will detach the database from current binlog connection and will retry establish new one to prevent other databases to wait for this database.
Default value: 10000
**Example**
```sql
CREATE DATABASE db1 ENGINE = MaterializedMySQL('host:port', 'db1', 'user', 'password') SETTINGS use_binlog_client=1, max_bytes_in_binlog_queue=33554432, max_milliseconds_to_wait_in_binlog_queue=1000
CREATE DATABASE db2 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1
```
If the event queue of database `db1` is full, the binlog connection will be waiting in `1000`ms and if the database is not able to consume the events, it will be detached from the connection to create another one.
NOTE: If the database `db1` has been detached from the shared connection and created new one, after the binlog connections for `db1` and `db2` have the same positions they will be merged to one. And `db1` and `db2` will use the same connection again.
## max_bytes_in_binlog_dispatcher_buffer
Defines the max bytes in the binlog dispatcher's buffer before it is flushed to attached binlog. The events from MySQL binlog connection are buffered before sending to attached databases. It increases the events throughput from the binlog to databases.
Default value: 1048576
## max_flush_milliseconds_in_binlog_dispatcher
Defines the max milliseconds in the binlog dispatcher's buffer to wait before it is flushed to attached binlog. If there are no events received from MySQL binlog connection for a while, after some time buffered events should be sent to the attached databases.
Default value: 1000
# Design
## The Binlog Events Dispatcher
Currently each MaterializedMySQL database opens its own connection to MySQL to subscribe to binlog events. There is a need to have only one connection and _dispatch_ the binlog events to all databases that replicate from the same MySQL instance.
## Each MaterializedMySQL Database Has Its Own Event Queue
To prevent slowing down other instances there should be an _event queue_ per MaterializedMySQL database to handle the events independently of the speed of other instances. The dispatcher reads an event from the binlog, and sends it to every MaterializedMySQL database that needs it. Each database handles its events in separate threads.
## Catching up
If several databases have the same binlog position, they can use the same dispatcher. If a newly created database (or one that has been detached for some time) requests events that have been already processed, we need to create another communication _channel_ to the binlog. We do this by creating another temporary dispatcher for such databases. When the new dispatcher _catches up with_ the old one, the new/temporary dispatcher is not needed anymore and all databases getting events from this dispatcher can be moved to the old one.
## Memory Limit
There is a _memory limit_ to control event queue memory consumption per MySQL Client. If a database is not able to handle events fast enough, and the event queue is getting full, we have the following options:
1. The dispatcher is blocked until the slowest database frees up space for new events. All other databases are waiting for the slowest one. (Preferred)
2. The dispatcher is _never_ blocked, but suspends incremental sync for the slow database and continues dispatching events to remained databases.
## Performance
A lot of CPU can be saved by not processing every event in every database. The binlog contains events for all databases, it is wasteful to distribute row events to a database that it will not process it, especially if there are a lot of databases. This requires some sort of per-database binlog filtering and buffering.
Currently all events are sent to all MaterializedMySQL databases but parsing the event which consumes CPU is up to the database.
# Detailed Design
1. If a client (e.g. database) wants to read a stream of the events from MySQL binlog, it creates a connection to remote binlog by host/user/password and _executed GTID set_ params.
2. If another client wants to read the events from the binlog but for different _executed GTID set_, it is **not** possible to reuse existing connection to MySQL, then need to create another connection to the same remote binlog. (_This is how it is implemented today_).
3. When these 2 connections get the same binlog positions, they read the same events. It is logical to drop duplicate connection and move all its users out. And now one connection dispatches binlog events to several clients. Obviously only connections to the same binlog should be merged.
## Classes
1. One connection can send (or dispatch) events to several clients and might be called `BinlogEventsDispatcher`.
2. Several dispatchers grouped by _user:password@host:port_ in `BinlogClient`. Since they point to the same binlog.
3. The clients should communicate only with public API from `BinlogClient`. The result of using `BinlogClient` is an object that implements `IBinlog` to read events from. This implementation of `IBinlog` must be compatible with old implementation `MySQLFlavor` -> when replacing old implementation by new one, the behavior must not be changed.
## SQL
```sql
-- create MaterializedMySQL databases that read the events from the binlog client
CREATE DATABASE db1_client1 ENGINE = MaterializedMySQL('host:port', 'db', 'user', 'password') SETTINGS use_binlog_client=1, max_bytes_in_binlog_queue=1024;
CREATE DATABASE db2_client1 ENGINE = MaterializedMySQL('host:port', 'db', 'user', 'password') SETTINGS use_binlog_client=1;
CREATE DATABASE db3_client1 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1;
CREATE DATABASE db4_client2 ENGINE = MaterializedMySQL('host2:port', 'db', 'user', 'password') SETTINGS use_binlog_client=1;
CREATE DATABASE db5_client3 ENGINE = MaterializedMySQL('host:port', 'db', 'user1', 'password') SETTINGS use_binlog_client=1;
CREATE DATABASE db6_old ENGINE = MaterializedMySQL('host:port', 'db', 'user1', 'password') SETTINGS use_binlog_client=0;
```
Databases `db1_client1`, `db2_client1` and `db3_client1` share one instance of `BinlogClient` since they have the same params. `BinlogClient` will create 3 connections to MySQL server thus 3 instances of `BinlogEventsDispatcher`, but if these connections would have the same binlog position, they should be merged to one connection. Means all clients will be moved to one dispatcher and others will be closed. Databases `db4_client2` and `db5_client3` would use 2 different independent `BinlogClient` instances. Database `db6_old` will use old implementation. NOTE: By default `use_binlog_client` is disabled. Setting `max_bytes_in_binlog_queue` defines the max allowed bytes in the binlog queue. By default, it is `1073741824` bytes. If number of bytes exceeds this limit, the dispatching will be stopped until the space will be freed for new events.
## Binlog Table Structure
To see the status of the all `BinlogClient` instances there is `system.mysql_binlogs` system table. It shows the list of all created and _alive_ `IBinlog` instances with information about its `BinlogEventsDispatcher` and `BinlogClient`.
Example:
```
SELECT * FROM system.mysql_binlogs FORMAT Vertical
Row 1:
──────
binlog_client_name: root@127.0.0.1:3306
name: test_Clickhouse1
mysql_binlog_name: binlog.001154
mysql_binlog_pos: 7142294
mysql_binlog_timestamp: 1660082447
mysql_binlog_executed_gtid_set: a9d88f83-c14e-11ec-bb36-244bfedf7766:1-30523304
dispatcher_name: Applier
dispatcher_mysql_binlog_name: binlog.001154
dispatcher_mysql_binlog_pos: 7142294
dispatcher_mysql_binlog_timestamp: 1660082447
dispatcher_mysql_binlog_executed_gtid_set: a9d88f83-c14e-11ec-bb36-244bfedf7766:1-30523304
size: 0
bytes: 0
max_bytes: 0
```
### Tests
Unit tests:
```
$ ./unit_tests_dbms --gtest_filter=MySQLBinlog.*
```
Integration tests:
```
$ pytest -s -vv test_materialized_mysql_database/test.py::test_binlog_client
```
Dumps events from the file
```
$ ./utils/check-mysql-binlog/check-mysql-binlog --binlog binlog.001392
```
Dumps events from the server
```
$ ./utils/check-mysql-binlog/check-mysql-binlog --host 127.0.0.1 --port 3306 --user root --password pass --gtid a9d88f83-c14e-11ec-bb36-244bfedf7766:1-30462856
```

2319
docs/en/operations/settings/settings-formats.md
View File

File diff suppressed because it is too large Load Diff

9753
docs/en/operations/settings/settings.md
View File

File diff suppressed because it is too large Load Diff

20
docs/en/operations/system-tables/azure_queue_settings.md Normal file
View File

@ -0,0 +1,20 @@
---
slug: /en/operations/system-tables/azure_queue_settings
---
# azure_queue_settings
Contains information about settings of [AzureQueue](../../engines/table-engines/integrations/azure-queue.md) tables.
Available from `24.10` server version.
Columns:
- `database` ([String](../../sql-reference/data-types/string.md)) — Table name.
- `table` ([String](../../sql-reference/data-types/string.md)) — Database name.
- `name` ([String](../../sql-reference/data-types/string.md)) — Setting name.
- `value` ([String](../../sql-reference/data-types/string.md)) — Setting value.
- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Whether the setting was explicitly defined in the config or explicitly changed.
- `description` ([String](../../sql-reference/data-types/string.md)) — Setting description.
- `alterable` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the setting can be changes via `ALTER TABLE ... MODIFY SETTING`.
- `0` — Current user can alter the setting.
- `1` — Current user can’t alter the setting.
- `type` ([String](../../sql-reference/data-types/string.md)) — Setting type (implementation specific string value).

8
docs/en/operations/system-tables/metrics.md
View File

@ -737,6 +737,14 @@ Number of sessions (connections) to ZooKeeper. Should be no more than one, becau
Number of watches (event subscriptions) in ZooKeeper.
### ConcurrencyControlAcquired
Total number of acquired CPU slots.
### ConcurrencyControlSoftLimit
Value of soft limit on number of CPU slots.
**See Also**
- [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics) — Contains periodically calculated metrics.

6
docs/en/operations/system-tables/part_log.md
View File

@ -13,10 +13,12 @@ The `system.part_log` table contains the following columns:
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Identifier of the `INSERT` query that created this data part.
- `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event that occurred with the data part. Can have one of the following values:
- `NewPart` — Inserting of a new data part.
- `MergeParts` — Merging of data parts.
- `MergePartsStart` — Merging of data parts has started.
- `MergeParts` — Merging of data parts has finished.
- `DownloadPart` — Downloading a data part.
- `RemovePart` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
- `MutatePart` — Mutating of a data part.
- `MutatePartStart` — Mutating of a data part has started.
- `MutatePart` — Mutating of a data part has finished.
- `MovePart` — Moving the data part from the one disk to another one.
- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values:
- `NotAMerge` — The current event has the type other than `MERGE_PARTS`.

20
docs/en/operations/system-tables/s3_queue_settings.md Normal file
View File

@ -0,0 +1,20 @@
---
slug: /en/operations/system-tables/s3_queue_settings
---
# s3_queue_settings
Contains information about settings of [S3Queue](../../engines/table-engines/integrations/s3queue.md) tables.
Available from `24.10` server version.
Columns:
- `database` ([String](../../sql-reference/data-types/string.md)) — Table name.
- `table` ([String](../../sql-reference/data-types/string.md)) — Database name.
- `name` ([String](../../sql-reference/data-types/string.md)) — Setting name.
- `value` ([String](../../sql-reference/data-types/string.md)) — Setting value.
- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Whether the setting was explicitly defined in the config or explicitly changed.
- `description` ([String](../../sql-reference/data-types/string.md)) — Setting description.
- `alterable` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the setting can be changes via `ALTER TABLE ... MODIFY SETTING`.
- `0` — Current user can alter the setting.
- `1` — Current user can’t alter the setting.
- `type` ([String](../../sql-reference/data-types/string.md)) — Setting type (implementation specific string value).

20
docs/en/operations/utilities/clickhouse-local.md
View File

@ -177,6 +177,26 @@ When you are ready to insert your files into ClickHouse, startup a ClickHouse se
:::
## Format Conversions
You can use `clickhouse-local` for converting data between different formats. Example:
``` bash
$ clickhouse-local --input-format JSONLines --output-format CSV --query "SELECT * FROM table" < data.json > data.csv
```
Formats are auto-detected from file extensions:
``` bash
$ clickhouse-local --query "SELECT * FROM table" < data.json > data.csv
```
As a shortcut, you can write it using the `--copy` argument:
``` bash
$ clickhouse-local --copy < data.json > data.csv
```
## Usage {#usage}
By default `clickhouse-local` has access to data of a ClickHouse server on the same host, and it does not depend on the server's configuration. It also supports loading server configuration using `--config-file` argument. For temporary data, a unique temporary data directory is created by default.

21
docs/en/sql-reference/aggregate-functions/parametric-functions.md
View File

@ -261,9 +261,10 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN)
- `window` — Length of the sliding window, it is the time interval between the first and the last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`.
- `mode` — It is an optional argument. One or more modes can be set.
- `'strict_deduplication'` — If the same condition holds for the sequence of events, then such repeating event interrupts further processing.
- `'strict_deduplication'` — If the same condition holds for the sequence of events, then such repeating event interrupts further processing. Note: it may work unexpectedly if several conditions hold for the same event.
- `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2.
- `'strict_increase'` — Apply conditions only to events with strictly increasing timestamps.
- `'strict_once'` — Count each event only once in the chain even if it meets the condition several times
**Returned value**
@ -490,7 +491,7 @@ Where:
## uniqUpTo(N)(x)
Calculates the number of different values of the argument up to a specified limit, `N`. If the number of different argument values is greater than `N`, this function returns `N` + 1, otherwise it calculates the exact value.
Calculates the number of different values of the argument up to a specified limit, `N`. If the number of different argument values is greater than `N`, this function returns `N` + 1, otherwise it calculates the exact value.
Recommended for use with small `N`s, up to 10. The maximum value of `N` is 100.
@ -522,7 +523,7 @@ This function behaves the same as [sumMap](../../sql-reference/aggregate-functio
- `keys`: [Array](../data-types/array.md) of keys.
- `values`: [Array](../data-types/array.md) of values.
**Returned Value**
**Returned Value**
- Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys.
@ -539,10 +540,10 @@ CREATE TABLE sum_map
)
ENGINE = Log
INSERT INTO sum_map VALUES
('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]),
INSERT INTO sum_map VALUES
('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]);
```
@ -572,7 +573,7 @@ This function behaves the same as [sumMap](../../sql-reference/aggregate-functio
- `keys`: [Array](../data-types/array.md) of keys.
- `values`: [Array](../data-types/array.md) of values.
**Returned Value**
**Returned Value**
- Returns a tuple of two arrays: keys in sorted order, and values ​​summed for the corresponding keys.
@ -591,10 +592,10 @@ CREATE TABLE sum_map
)
ENGINE = Log
INSERT INTO sum_map VALUES
('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]),
INSERT INTO sum_map VALUES
('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]),
('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]);
```

2
docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md
View File

@ -18,7 +18,7 @@ Parameters:
Returned values:
Constants `(a, b)` of the resulting line `y = a*x + b`.
Constants `(k, b)` of the resulting line `y = k*x + b`.
**Examples**

190
docs/en/sql-reference/ansi.md
View File

@ -1,190 +0,0 @@
---
slug: /en/sql-reference/ansi
sidebar_position: 40
sidebar_label: ANSI Compatibility
title: "ANSI SQL Compatibility of ClickHouse SQL Dialect"
---
:::note
This article relies on Table 38, “Feature taxonomy and definition for mandatory features”, Annex F of [ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8).
:::
## Differences in Behaviour
The following table lists cases when query feature works in ClickHouse, but behaves not as specified in ANSI SQL.
| Feature ID | Feature Name | Difference |
|------------|-----------------------------|-----------------------------------------------------------------------------------------------------------|
| E011 | Numeric data types | Numeric literal with period is interpreted as approximate (`Float64`) instead of exact (`Decimal`) |
| E051-05 | Select items can be renamed | Item renames have a wider visibility scope than just the SELECT result |
| E141-01 | NOT NULL constraints | `NOT NULL` is implied for table columns by default |
| E011-04 | Arithmetic operators | ClickHouse overflows instead of checked arithmetic and changes the result data type based on custom rules |
## Feature Status
| Feature ID | Feature Name | Status | Comment |
|------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **E011** | **Numeric data types** | <span class="text-warning">Partial</span> | |
| E011-01 | INTEGER and SMALLINT data types | <span class="text-success">Yes</span> | |
| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | <span class="text-success">Yes</span> | |
| E011-03 | DECIMAL and NUMERIC data types | <span class="text-success">Yes</span> | |
| E011-04 | Arithmetic operators | <span class="text-success">Yes</span> | |
| E011-05 | Numeric comparison | <span class="text-success">Yes</span> | |
| E011-06 | Implicit casting among the numeric data types | <span class="text-danger">No</span> | ANSI SQL allows arbitrary implicit cast between numeric types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
| **E021** | **Character string types** | <span class="text-warning">Partial</span> | |
| E021-01 | CHARACTER data type | <span class="text-success">Yes</span> | |
| E021-02 | CHARACTER VARYING data type | <span class="text-success">Yes</span> | |
| E021-03 | Character literals | <span class="text-success">Yes</span> | |
| E021-04 | CHARACTER_LENGTH function | <span class="text-warning">Partial</span> | No `USING` clause |
| E021-05 | OCTET_LENGTH function | <span class="text-danger">No</span> | `LENGTH` behaves similarly |
| E021-06 | SUBSTRING | <span class="text-warning">Partial</span> | No support for `SIMILAR` and `ESCAPE` clauses, no `SUBSTRING_REGEX` variant |
| E021-07 | Character concatenation | <span class="text-warning">Partial</span> | No `COLLATE` clause |
| E021-08 | UPPER and LOWER functions | <span class="text-success">Yes</span> | |
| E021-09 | TRIM function | <span class="text-success">Yes</span> | |
| E021-10 | Implicit casting among the fixed-length and variable-length character string types | <span class="text-warning">Partial</span> | ANSI SQL allows arbitrary implicit cast between string types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
| E021-11 | POSITION function | <span class="text-warning">Partial</span> | No support for `IN` and `USING` clauses, no `POSITION_REGEX` variant |
| E021-12 | Character comparison | <span class="text-success">Yes</span> | |
| **E031** | **Identifiers** | <span class="text-warning">Partial</span>| |
| E031-01 | Delimited identifiers | <span class="text-warning">Partial</span> | Unicode literal support is limited |
| E031-02 | Lower case identifiers | <span class="text-success">Yes</span> | |
| E031-03 | Trailing underscore | <span class="text-success">Yes</span> | |
| **E051** | **Basic query specification** | <span class="text-warning">Partial</span>| |
| E051-01 | SELECT DISTINCT | <span class="text-success">Yes</span> | |
| E051-02 | GROUP BY clause | <span class="text-success">Yes</span> | |
| E051-04 | GROUP BY can contain columns not in `<select list>` | <span class="text-success">Yes</span> | |
| E051-05 | Select items can be renamed | <span class="text-success">Yes</span> | |
| E051-06 | HAVING clause | <span class="text-success">Yes</span> | |
| E051-07 | Qualified \* in select list | <span class="text-success">Yes</span> | |
| E051-08 | Correlation name in the FROM clause | <span class="text-success">Yes</span> | |
| E051-09 | Rename columns in the FROM clause | <span class="text-danger">No</span> | |
| **E061** | **Basic predicates and search conditions** | <span class="text-warning">Partial</span> | |
| E061-01 | Comparison predicate | <span class="text-success">Yes</span> | |
| E061-02 | BETWEEN predicate | <span class="text-warning">Partial</span> | No `SYMMETRIC` and `ASYMMETRIC` clause |
| E061-03 | IN predicate with list of values | <span class="text-success">Yes</span> | |
| E061-04 | LIKE predicate | <span class="text-success">Yes</span> | |
| E061-05 | LIKE predicate: ESCAPE clause | <span class="text-danger">No</span> | |
| E061-06 | NULL predicate | <span class="text-success">Yes</span> | |
| E061-07 | Quantified comparison predicate | <span class="text-danger">No</span> | |
| E061-08 | EXISTS predicate | <span class="text-danger">No</span> | |
| E061-09 | Subqueries in comparison predicate | <span class="text-success">Yes</span> | |
| E061-11 | Subqueries in IN predicate | <span class="text-success">Yes</span> | |
| E061-12 | Subqueries in quantified comparison predicate | <span class="text-danger">No</span> | |
| E061-13 | Correlated subqueries | <span class="text-danger">No</span> | |
| E061-14 | Search condition | <span class="text-success">Yes</span> | |
| **E071** | **Basic query expressions** | <span class="text-warning">Partial</span> | |
| E071-01 | UNION DISTINCT table operator | <span class="text-success">Yes</span> | |
| E071-02 | UNION ALL table operator | <span class="text-success">Yes</span> | |
| E071-03 | EXCEPT DISTINCT table operator | <span class="text-danger">No</span> | |
| E071-05 | Columns combined via table operators need not have exactly the same data type | <span class="text-success">Yes</span> | |
| E071-06 | Table operators in subqueries | <span class="text-success">Yes</span> | |
| **E081** | **Basic privileges** | <span class="text-success">Yes</span> |
| E081-01 | SELECT privilege at the table level | <span class="text-success">Yes</span> |
| E081-02 | DELETE privilege | |
| E081-03 | INSERT privilege at the table level | <span class="text-success">Yes</span> |
| E081-04 | UPDATE privilege at the table level | <span class="text-success">Yes</span> |
| E081-05 | UPDATE privilege at the column level | |
| E081-06 | REFERENCES privilege at the table level | | |
| E081-07 | REFERENCES privilege at the column level | | |
| E081-08 | WITH GRANT OPTION | <span class="text-success">Yes</span> | |
| E081-09 | USAGE privilege | | |
| E081-10 | EXECUTE privilege | | |
| **E091** | **Set functions** |<span class="text-success">Yes</span> |
| E091-01 | AVG | <span class="text-success">Yes</span> | |
| E091-02 | COUNT | <span class="text-success">Yes</span> | |
| E091-03 | MAX | <span class="text-success">Yes</span> | |
| E091-04 | MIN | <span class="text-success">Yes</span> | |
| E091-05 | SUM | <span class="text-success">Yes</span> | |
| E091-06 | ALL quantifier | <span class="text-success">Yes</span> | |
| E091-07 | DISTINCT quantifier | <span class="text-success">Yes</span> | Not all aggregate functions supported |
| **E101** | **Basic data manipulation** | <span class="text-warning">Partial</span> | |
| E101-01 | INSERT statement | <span class="text-success">Yes</span> | Note: primary key in ClickHouse does not imply the `UNIQUE` constraint |
| E101-03 | Searched UPDATE statement | <span class="text-warning">Partial</span> | There’s an `ALTER UPDATE` statement for batch data modification |
| E101-04 | Searched DELETE statement | <span class="text-warning">Partial</span> | There’s an `ALTER DELETE` statement for batch data removal |
| **E111** | **Single row SELECT statement** | <span class="text-danger">No</span> | |
| **E121** | **Basic cursor support** | <span class="text-danger">No</span> | |
| E121-01 | DECLARE CURSOR | <span class="text-danger">No</span> | |
| E121-02 | ORDER BY columns need not be in select list | <span class="text-success">Yes</span> | |
| E121-03 | Value expressions in ORDER BY clause | <span class="text-success">Yes</span> | |
| E121-04 | OPEN statement | <span class="text-danger">No</span> | |
| E121-06 | Positioned UPDATE statement | <span class="text-danger">No</span> | |
| E121-07 | Positioned DELETE statement | <span class="text-danger">No</span> | |
| E121-08 | CLOSE statement | <span class="text-danger">No</span> | |
| E121-10 | FETCH statement: implicit NEXT | <span class="text-danger">No</span> | |
| E121-17 | WITH HOLD cursors | <span class="text-danger">No</span> | |
| **E131** | **Null value support (nulls in lieu of values)** | <span class="text-success">Yes</span> | Some restrictions apply |
| **E141** | **Basic integrity constraints** | <span class="text-warning">Partial</span> | |
| E141-01 | NOT NULL constraints | <span class="text-success">Yes</span> | Note: `NOT NULL` is implied for table columns by default |
| E141-02 | UNIQUE constraint of NOT NULL columns | <span class="text-danger">No</span> | |
| E141-03 | PRIMARY KEY constraints | <span class="text-warning">Partial</span> | |
| E141-04 | Basic FOREIGN KEY constraint with the NO ACTION default for both referential delete action and referential update action | <span class="text-danger">No</span> | |
| E141-06 | CHECK constraint | <span class="text-success">Yes</span> | |
| E141-07 | Column defaults | <span class="text-success">Yes</span> | |
| E141-08 | NOT NULL inferred on PRIMARY KEY | <span class="text-success">Yes</span> | |
| E141-10 | Names in a foreign key can be specified in any order | <span class="text-danger">No</span> | |
| **E151** | **Transaction support** | <span class="text-danger">No</span> | |
| E151-01 | COMMIT statement | <span class="text-danger">No</span> | |
| E151-02 | ROLLBACK statement | <span class="text-danger">No</span> | |
| **E152** | **Basic SET TRANSACTION statement** | <span class="text-danger">No</span> | |
| E152-01 | SET TRANSACTION statement: ISOLATION LEVEL SERIALIZABLE clause | <span class="text-danger">No</span> | |
| E152-02 | SET TRANSACTION statement: READ ONLY and READ WRITE clauses | <span class="text-danger">No</span> | |
| **E153** | **Updatable queries with subqueries** | <span class="text-success">Yes</span> | |
| **E161** | **SQL comments using leading double minus** | <span class="text-success">Yes</span> | |
| **E171** | **SQLSTATE support** | <span class="text-danger">No</span> | |
| **E182** | **Host language binding** | <span class="text-danger">No</span> | |
| **F031** | **Basic schema manipulation** | <span class="text-warning">Partial</span>| |
| F031-01 | CREATE TABLE statement to create persistent base tables | <span class="text-warning">Partial</span> | No `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` clauses and no support for user resolved data types |
| F031-02 | CREATE VIEW statement | <span class="text-warning">Partial</span> | No `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` clauses and no support for user resolved data types |
| F031-03 | GRANT statement | <span class="text-success">Yes</span> | |
| F031-04 | ALTER TABLE statement: ADD COLUMN clause | <span class="text-success">Yes</span> | No support for `GENERATED` clause and system time period |
| F031-13 | DROP TABLE statement: RESTRICT clause | <span class="text-danger">No</span> | |
| F031-16 | DROP VIEW statement: RESTRICT clause | <span class="text-danger">No</span> | |
| F031-19 | REVOKE statement: RESTRICT clause | <span class="text-danger">No</span> | |
| **F041** | **Basic joined table** | <span class="text-warning">Partial</span> | |
| F041-01 | Inner join (but not necessarily the INNER keyword) | <span class="text-success">Yes</span> | |
| F041-02 | INNER keyword | <span class="text-success">Yes</span> | |
| F041-03 | LEFT OUTER JOIN | <span class="text-success">Yes</span> | |
| F041-04 | RIGHT OUTER JOIN | <span class="text-success">Yes</span> | |
| F041-05 | Outer joins can be nested | <span class="text-success">Yes</span> | |
| F041-07 | The inner table in a left or right outer join can also be used in an inner join | <span class="text-success">Yes</span> | |
| F041-08 | All comparison operators are supported (rather than just =) | <span class="text-danger">No</span> | |
| **F051** | **Basic date and time** | <span class="text-warning">Partial</span> | |
| F051-01 | DATE data type (including support of DATE literal) | <span class="text-success">Yes</span> | |
| F051-02 | TIME data type (including support of TIME literal) with fractional seconds precision of at least 0 | <span class="text-danger">No</span> | |
| F051-03 | TIMESTAMP data type (including support of TIMESTAMP literal) with fractional seconds precision of at least 0 and 6 | <span class="text-success">Yes</span> | |
| F051-04 | Comparison predicate on DATE, TIME, and TIMESTAMP data types | <span class="text-success">Yes</span> | |
| F051-05 | Explicit CAST between datetime types and character string types | <span class="text-success">Yes</span> | |
| F051-06 | CURRENT_DATE | <span class="text-danger">No</span> | `today()` is similar |
| F051-07 | LOCALTIME | <span class="text-danger">No</span> | `now()` is similar |
| F051-08 | LOCALTIMESTAMP | <span class="text-danger">No</span> | |
| **F081** | **UNION and EXCEPT in views** | <span class="text-warning">Partial</span> | |
| **F131** | **Grouped operations** | <span class="text-warning">Partial</span> | |
| F131-01 | WHERE, GROUP BY, and HAVING clauses supported in queries with grouped views | <span class="text-success">Yes</span> | |
| F131-02 | Multiple tables supported in queries with grouped views | <span class="text-success">Yes</span> | |
| F131-03 | Set functions supported in queries with grouped views | <span class="text-success">Yes</span> | |
| F131-04 | Subqueries with GROUP BY and HAVING clauses and grouped views | <span class="text-success">Yes</span> | |
| F131-05 | Single row SELECT with GROUP BY and HAVING clauses and grouped views | <span class="text-danger">No</span> | |
| **F181** | **Multiple module support** | <span class="text-danger">No</span> | |
| **F201** | **CAST function** | <span class="text-success">Yes</span> | |
| **F221** | **Explicit defaults** | <span class="text-danger">No</span> | |
| **F261** | **CASE expression** | <span class="text-success">Yes</span> | |
| F261-01 | Simple CASE | <span class="text-success">Yes</span> | |
| F261-02 | Searched CASE | <span class="text-success">Yes</span> | |
| F261-03 | NULLIF | <span class="text-success">Yes</span> | |
| F261-04 | COALESCE | <span class="text-success">Yes</span> | |
| **F311** | **Schema definition statement** | <span class="text-warning">Partial</span> | |
| F311-01 | CREATE SCHEMA | <span class="text-warning">Partial</span> | See CREATE DATABASE |
| F311-02 | CREATE TABLE for persistent base tables | <span class="text-success">Yes</span> | |
| F311-03 | CREATE VIEW | <span class="text-success">Yes</span> | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | <span class="text-danger">No</span> | |
| F311-05 | GRANT statement | <span class="text-success">Yes</span> | |
| **F471** | **Scalar subquery values** | <span class="text-success">Yes</span> | |
| **F481** | **Expanded NULL predicate** | <span class="text-success">Yes</span> | |
| **F812** | **Basic flagging** | <span class="text-danger">No</span> | |
| **S011** | **Distinct data types** | | |
| **T321** | **Basic SQL-invoked routines** | <span class="text-danger">No</span> | |
| T321-01 | User-defined functions with no overloading | <span class="text-danger">No</span> | |
| T321-02 | User-defined stored procedures with no overloading | <span class="text-danger">No</span> | |
| T321-03 | Function invocation | <span class="text-danger">No</span> | |
| T321-04 | CALL statement | <span class="text-danger">No</span> | |
| T321-05 | RETURN statement | <span class="text-danger">No</span> | |
| **T631** | **IN predicate with one list element** | <span class="text-success">Yes</span> | |

37
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -2933,7 +2933,42 @@ The same as ‘today() - 1’.
## timeSlot
Rounds the time to the half hour.
Round the time to the start of a half-an-hour length interval.
**Syntax**
```sql
timeSlot(time[, time_zone])
```
**Arguments**
- `time` — Time to round to the start of a half-an-hour length interval. [DateTime](../data-types/datetime.md)/[Date32](../data-types/date32.md)/[DateTime64](../data-types/datetime64.md).
- `time_zone` — A String type const value or an expression representing the time zone. [String](../data-types/string.md).
:::note
Though this function can take values of the extended types `Date32` and `DateTime64` as an argument, passing it a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results.
:::
**Return type**
- Returns the time rounded to the start of a half-an-hour length interval. [DateTime](../data-types/datetime.md).
**Example**
Query:
```sql
SELECT timeSlot(toDateTime('2000-01-02 03:04:05', 'UTC'));
```
Result:
```response
┌─timeSlot(toDateTime('2000-01-02 03:04:05', 'UTC'))─┐
│ 2000-01-02 03:00:00 │
└────────────────────────────────────────────────────┘
```
## toYYYYMM

238
docs/en/sql-reference/functions/ip-address-functions.md
View File

@ -244,24 +244,42 @@ SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32);
└────────────────────────────────────────────────────────────────────────┘
```
## toIPv4(string)
## toIPv4
An alias to `IPv4StringToNum()` that takes a string form of IPv4 address and returns value of [IPv4](../data-types/ipv4.md) type, which is binary equal to value returned by `IPv4StringToNum()`.
Like [`IPv4StringToNum`](##IPv4NumToString(num)) but takes a string form of IPv4 address and returns value of [IPv4](../data-types/ipv4.md) type.
**Syntax**
```sql
toIPv4(string)
```
**Arguments**
- `string` — IPv4 address. [String](../data-types/string.md).
**Returned value**
- `string` converted to the IPv4 address. [IPv4](../data-types/ipv4.md).
**Examples**
Query:
``` sql
WITH
'171.225.130.45' as IPv4_string
SELECT
toTypeName(IPv4StringToNum(IPv4_string)),
toTypeName(toIPv4(IPv4_string))
SELECT toIPv4('171.225.130.45');
```
Result:
``` text
┌─toTypeName(IPv4StringToNum(IPv4_string))─┬─toTypeName(toIPv4(IPv4_string))─┐
│ UInt32 │ IPv4 │
└──────────────────────────────────────────┴─────────────────────────────────┘
┌─toIPv4('171.225.130.45')─┐
│ 171.225.130.45 │
└──────────────────────────┘
```
Query:
``` sql
WITH
'171.225.130.45' as IPv4_string
@ -270,91 +288,124 @@ SELECT
hex(toIPv4(IPv4_string))
```
Result:
``` text
┌─hex(IPv4StringToNum(IPv4_string))─┬─hex(toIPv4(IPv4_string))─┐
│ ABE1822D │ ABE1822D │
└───────────────────────────────────┴──────────────────────────┘
```
## toIPv4OrDefault(string)
## toIPv4OrDefault
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4).
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4), or the provided IPv4 default.
**Syntax**
```sql
toIPv4OrDefault(value)
toIPv4OrDefault(string[, default])
```
**Arguments**
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
- `value` — IP address. [String](../data-types/string.md).
- `default` (optional) — The value to return if `string` has an invalid format. [IPv4](../data-types/ipv4.md).
**Returned value**
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
- `string` converted to the current IPv4 address. [String](../data-types/string.md).
**Example**
Query:
```sql
WITH
'::ffff:127.0.0.1' AS valid_IPv6_string,
'fe80:2030:31:24' AS invalid_IPv6_string
SELECT
toIPv4OrDefault('192.168.0.1') AS s1,
toIPv4OrDefault('192.168.0') AS s2
toIPv4OrDefault(valid_IPv6_string) AS valid,
toIPv4OrDefault(invalid_IPv6_string) AS default,
toIPv4OrDefault(invalid_IPv6_string, toIPv4('1.1.1.1')) AS provided_default;
```
Result:
```response
┌─s1──────────┬─s2──────┐
│ 192.168.0.1 │ 0.0.0.0 │
└─────────────┴─────────┘
┌─valid───┬─default─┬─provided_default─┐
│ 0.0.0.0 │ 0.0.0.0 │ 1.1.1.1 │
└─────────┴─────────┴──────────────────┘
```
## toIPv4OrNull(string)
## toIPv4OrNull
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns null.
Same as [`toIPv4`](#toipv4), but if the IPv4 address has an invalid format, it returns null.
**Syntax**
```sql
toIPv4OrNull(value)
toIPv4OrNull(string)
```
**Arguments**
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
- `string` — IP address. [String](../data-types/string.md).
**Returned value**
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
- `string` converted to the current IPv4 address, or null if `string` is an invalid address. [String](../data-types/string.md).
**Example**
Query:
```sql
SELECT
toIPv4OrNull('192.168.0.1') AS s1,
toIPv4OrNull('192.168.0') AS s2
``` sql
WITH 'fe80:2030:31:24' AS invalid_IPv6_string
SELECT toIPv4OrNull(invalid_IPv6_string);
```
Result:
```response
┌─s1──────────┬─s2───┐
│ 192.168.0.1 │ ᴺᵁᴸᴸ │
└─────────────┴──────┘
``` text
┌─toIPv4OrNull(invalid_IPv6_string)─┐
│ ᴺᵁᴸᴸ │
└───────────────────────────────────┘
```
## toIPv6OrDefault(string)
## toIPv4OrZero
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns `::` (0 IPv6).
Same as [`toIPv4`](#toipv4), but if the IPv4 address has an invalid format, it returns `0.0.0.0`.
## toIPv6OrNull(string)
**Syntax**
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null.
```sql
toIPv4OrZero(string)
```
**Arguments**
- `string` — IP address. [String](../data-types/string.md).
**Returned value**
- `string` converted to the current IPv4 address, or `0.0.0.0` if `string` is an invalid address. [String](../data-types/string.md).
**Example**
Query:
``` sql
WITH 'Not an IP address' AS invalid_IPv6_string
SELECT toIPv4OrZero(invalid_IPv6_string);
```
Result:
``` text
┌─toIPv4OrZero(invalid_IPv6_string)─┐
│ 0.0.0.0 │
└───────────────────────────────────┘
```
## toIPv6
@ -371,7 +422,7 @@ toIPv6(string)
**Argument**
- `string` — IP address. [String](../data-types/string.md)
- `string` — IP address. [String](../data-types/string.md).
**Returned value**
@ -410,6 +461,117 @@ Result:
└─────────────────────┘
```
## toIPv6OrDefault
Same as [`toIPv6`](#toipv6), but if the IPv6 address has an invalid format, it returns `::` (0 IPv6) or the provided IPv6 default.
**Syntax**
```sql
toIPv6OrDefault(string[, default])
```
**Argument**
- `string` — IP address. [String](../data-types/string.md).
- `default` (optional) — The value to return if `string` has an invalid format. [IPv6](../data-types/ipv6.md).
**Returned value**
- IPv6 address [IPv6](../data-types/ipv6.md), otherwise `::` or the provided optional default if `string` has an invalid format.
**Example**
Query:
``` sql
WITH
'127.0.0.1' AS valid_IPv4_string,
'127.0.0.1.6' AS invalid_IPv4_string
SELECT
toIPv6OrDefault(valid_IPv4_string) AS valid,
toIPv6OrDefault(invalid_IPv4_string) AS default,
toIPv6OrDefault(invalid_IPv4_string, toIPv6('1.1.1.1')) AS provided_default
```
Result:
``` text
┌─valid────────────┬─default─┬─provided_default─┐
│ ::ffff:127.0.0.1 │ :: │ ::ffff:1.1.1.1 │
└──────────────────┴─────────┴──────────────────┘
```
## toIPv6OrNull
Same as [`toIPv6`](#toipv6), but if the IPv6 address has an invalid format, it returns null.
**Syntax**
```sql
toIPv6OrNull(string)
```
**Argument**
- `string` — IP address. [String](../data-types/string.md).
**Returned value**
- IP address. [IPv6](../data-types/ipv6.md), or null if `string` is not a valid format.
**Example**
Query:
``` sql
WITH '127.0.0.1.6' AS invalid_IPv4_string
SELECT toIPv6OrNull(invalid_IPv4_string);
```
Result:
``` text
┌─toIPv6OrNull(invalid_IPv4_string)─┐
│ ᴺᵁᴸᴸ │
└───────────────────────────────────┘
```
## toIPv6OrZero
Same as [`toIPv6`](#toipv6), but if the IPv6 address has an invalid format, it returns `::`.
**Syntax**
```sql
toIPv6OrZero(string)
```
**Argument**
- `string` — IP address. [String](../data-types/string.md).
**Returned value**
- IP address. [IPv6](../data-types/ipv6.md), or `::` if `string` is not a valid format.
**Example**
Query:
``` sql
WITH '127.0.0.1.6' AS invalid_IPv4_string
SELECT toIPv6OrZero(invalid_IPv4_string);
```
Result:
``` text
┌─toIPv6OrZero(invalid_IPv4_string)─┐
│ :: │
└───────────────────────────────────┘
```
## IPv6StringToNumOrDefault(s)
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns 0.

34
docs/en/sql-reference/functions/other-functions.md
View File

@ -4390,3 +4390,37 @@ Result:
1. │ ['{ArraySizes}','{ArrayElements, TupleElement(keys), Regular}','{ArrayElements, TupleElement(values), Regular}'] │
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## globalVariable
Takes a constant string argument and returns the value of the global variable with that name. This function is intended for compatibility with MySQL and not needed or useful for normal operation of ClickHouse. Only few dummy global variables are defined.
**Syntax**
```sql
globalVariable(name)
```
**Arguments**
- `name` — Global variable name. [String](../data-types/string.md).
**Returned value**
- Returns the value of variable `name`.
**Example**
Query:
```sql
SELECT globalVariable('max_allowed_packet');
```
Result:
```response
┌─globalVariable('max_allowed_packet')─┐
│ 67108864 │
└──────────────────────────────────────┘
```

42
docs/en/sql-reference/functions/random-functions.md
View File

@ -24,7 +24,7 @@ Returns a random UInt32 number with uniform distribution.
Uses a linear congruential generator with an initial state obtained from the system, which means that while it appears random, it's not truly random and can be predictable if the initial state is known. For scenarios where true randomness is crucial, consider using alternative methods like system-level calls or integrating with external libraries.
### Syntax
**Syntax**
```sql
rand()
@ -32,15 +32,15 @@ rand()
Alias: `rand32`
### Arguments
**Arguments**
None.
### Returned value
**Returned value**
Returns a number of type UInt32.
### Example
**Example**
```sql
SELECT rand();
@ -54,23 +54,23 @@ SELECT rand();
Returns a random UInt64 integer (UInt64) number
### Syntax
**Syntax**
```sql
rand64()
```
### Arguments
**Arguments**
None.
### Returned value
**Arguments**
Returns a number UInt64 number with uniform distribution.
Uses a linear congruential generator with an initial state obtained from the system, which means that while it appears random, it's not truly random and can be predictable if the initial state is known. For scenarios where true randomness is crucial, consider using alternative methods like system-level calls or integrating with external libraries.
### Example
**Example**
```sql
SELECT rand64();
@ -84,21 +84,21 @@ SELECT rand64();
Returns a random Float64 number.
### Syntax
**Syntax**
```sql
randCanonical()
```
### Arguments
**Arguments**
None.
### Returned value
**Arguments**
Returns a Float64 value between 0 (inclusive) and 1 (exclusive).
### Example
**Example**
```sql
SELECT randCanonical();
@ -112,25 +112,25 @@ SELECT randCanonical();
Generates a single constant column filled with a random value. Unlike `rand`, this function ensures the same random value appears in every row of the generated column, making it useful for scenarios requiring a consistent random seed across rows in a single query.
### Syntax
**Syntax**
```sql
randConstant([x]);
```
### Arguments
**Arguments**
- **[x] (Optional):** An optional expression that influences the generated random value. Even if provided, the resulting value will still be constant within the same query execution. Different queries using the same expression will likely generate different constant values.
### Returned value
**Arguments**
Returns a column of type UInt32 containing the same random value in each row.
### Implementation details
**Implementation details**
The actual output will be different for each query execution, even with the same optional expression. The optional parameter may not significantly change the generated value compared to using `randConstant` alone.
### Examples
**Example**
```sql
SELECT randConstant() AS random_value;
@ -156,22 +156,22 @@ SELECT randConstant(10) AS random_value;
Returns a random Float64 drawn uniformly from interval [`min`, `max`].
### Syntax
**Syntax**
```sql
randUniform(min, max)
```
### Arguments
**Arguments**
- `min` - `Float64` - left boundary of the range,
- `max` - `Float64` - right boundary of the range.
### Returned value
**Arguments**
A random number of type [Float64](../data-types/float.md).
### Example
**Example**
```sql
SELECT randUniform(5.5, 10) FROM numbers(5)

10
docs/en/sql-reference/functions/string-functions.md
View File

@ -1096,7 +1096,7 @@ convertCharset(s, from, to)
## base58Encode
Encodes a String using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) in the "Bitcoin" alphabet.
Encodes a string using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) in the "Bitcoin" alphabet.
**Syntax**
@ -1110,7 +1110,7 @@ base58Encode(plaintext)
**Returned value**
- A string containing the encoded value of the argument. [String](../data-types/string.md).
- A string containing the encoded value of the argument. [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md).
**Example**
@ -1128,7 +1128,7 @@ Result:
## base58Decode
Accepts a String and decodes it using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) encoding scheme using "Bitcoin" alphabet.
Accepts a string and decodes it using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) encoding scheme using "Bitcoin" alphabet.
**Syntax**
@ -1138,7 +1138,7 @@ base58Decode(encoded)
**Arguments**
- `encoded` — [String](../data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, an exception is thrown.
- `encoded` — [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md). If the string is not a valid Base58-encoded value, an exception is thrown.
**Returned value**
@ -1170,7 +1170,7 @@ tryBase58Decode(encoded)
**Parameters**
- `encoded`: [String](../data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, returns an empty string in case of error.
- `encoded`: [String](../data-types/string.md) or [FixedString](../data-types/fixedstring.md). If the string is not a valid Base58-encoded value, returns an empty string in case of error.
**Returned value**

6
docs/en/sql-reference/functions/type-conversion-functions.md
View File

@ -5261,9 +5261,9 @@ SELECT toFixedString('foo', 8) AS s;
Result:
```response
┌─s─────────────┬─s_cut─┐
│ foo\0\0\0\0\0 │ foo │
└───────────────┴───────┘
┌─s─────────────┐
│ foo\0\0\0\0\0 │
└───────────────┘
```
## toStringCutToZero

8
docs/en/sql-reference/statements/insert-into.md
View File

@ -121,6 +121,14 @@ However, you can delete old data using `ALTER TABLE ... DROP PARTITION`.
To insert a default value instead of `NULL` into a column with not nullable data type, enable [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default) setting.
`INSERT` also supports CTE(common table expression). For example, the following two statements are equivalent:
``` sql
INSERT INTO x WITH y AS (SELECT * FROM numbers(10)) SELECT * FROM y;
WITH y AS (SELECT * FROM numbers(10)) INSERT INTO x SELECT * FROM y;
```
## Inserting Data from a File
**Syntax**

4
docs/en/sql-reference/table-functions/generate.md
View File

@ -18,9 +18,9 @@ generateRandom(['name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_str
- `name` — Name of corresponding column.
- `TypeName` — Type of corresponding column.
- `max_array_length` — Maximum elements for all generated arrays or maps. Defaults to `10`.
- `max_string_length` — Maximum string length for all generated strings. Defaults to `10`.
- `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated.
- `max_string_length` — Maximum string length for all generated strings. Defaults to `10`.
- `max_array_length` — Maximum elements for all generated arrays or maps. Defaults to `10`.
**Returned Value**

9
docs/en/sql-reference/table-functions/iceberg.md
View File

@ -6,7 +6,7 @@ sidebar_label: iceberg
# iceberg Table Function
Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure or locally stored.
Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS or locally stored.
## Syntax
@ -17,13 +17,16 @@ icebergS3(named_collection[, option=value [,..]])
icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method])
icebergAzure(named_collection[, option=value [,..]])
icebergHDFS(path_to_table, [,format] [,compression_method])
icebergHDFS(named_collection[, option=value [,..]])
icebergLocal(path_to_table, [,format] [,compression_method])
icebergLocal(named_collection[, option=value [,..]])
```
## Arguments
Description of the arguments coincides with description of arguments in table functions `s3`, `azureBlobStorage` and `file` correspondingly.
Description of the arguments coincides with description of arguments in table functions `s3`, `azureBlobStorage`, `HDFS` and `file` correspondingly.
`format` stands for the format of data files in the Iceberg table.
**Returned value**
@ -36,7 +39,7 @@ SELECT * FROM icebergS3('http://test.s3.amazonaws.com/clickhouse-bucket/test_tab
```
:::important
ClickHouse currently supports reading v1 and v2 of the Iceberg format via the `icebergS3`, `icebergAzure` and `icebergLocal` table functions and `IcebergS3`, `icebergAzure` ans `icebergLocal` table engines.
ClickHouse currently supports reading v1 and v2 of the Iceberg format via the `icebergS3`, `icebergAzure`, `icebergHDFS` and `icebergLocal` table functions and `IcebergS3`, `icebergAzure`, `IcebergHDFS` and `IcebergLocal` table engines.
:::
## Defining a named collection

2
docs/mkdocs.yml
View File

@ -1,2 +0,0 @@
# Just an empty yaml file. Keep it alone.
{}

187
docs/ru/engines/database-engines/materialized-mysql.md
View File

@ -7,190 +7,3 @@ sidebar_label: "[experimental] MaterializedMySQL"
# [экспериментальный] MaterializedMySQL {#materialized-mysql}
**Это экспериментальный движок, который не следует использовать в продакшене.**
Создает базу данных ClickHouse со всеми таблицами, существующими в MySQL, и всеми данными в этих таблицах.
Сервер ClickHouse работает как реплика MySQL. Он читает файл binlog и выполняет DDL and DML-запросы.
## Создание базы данных {#creating-a-database}
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
```
**Параметры движка**
- `host:port` — адрес сервера MySQL.
- `database` — имя базы данных на удалённом сервере.
- `user` — пользователь MySQL.
- `password` — пароль пользователя.
**Настройки движка**
- `max_rows_in_buffer` — максимальное количество строк, содержимое которых может кешироваться в памяти (для одной таблицы и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `65 505`.
- `max_bytes_in_buffer` — максимальное количество байтов, которое разрешено кешировать в памяти (для одной таблицы и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `1 048 576`.
- `max_rows_in_buffers` — максимальное количество строк, содержимое которых может кешироваться в памяти (для базы данных и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `65 505`.
- `max_bytes_in_buffers` — максимальное количество байтов, которое разрешено кешировать данным в памяти (для базы данных и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `1 048 576`.
- `max_flush_data_time` — максимальное время в миллисекундах, в течение которого разрешено кешировать данные в памяти (для базы данных и данных кеша, которые невозможно запросить). При превышении количества указанного периода, данные будут материализованы. Значение по умолчанию: `1000`.
- `max_wait_time_when_mysql_unavailable` — интервал между повторными попытками, если MySQL недоступен. Указывается в миллисекундах. Отрицательное значение отключает повторные попытки. Значение по умолчанию: `1000`.
- `allows_query_when_mysql_lost` — признак, разрешен ли запрос к материализованной таблице при потере соединения с MySQL. Значение по умолчанию: `0` (`false`).
```sql
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
SETTINGS
allows_query_when_mysql_lost=true,
max_wait_time_when_mysql_unavailable=10000;
```
**Настройки на стороне MySQL-сервера**
Для правильной работы `MaterializedMySQL` следует обязательно указать на сервере MySQL следующие параметры конфигурации:
- `default_authentication_plugin = mysql_native_password` — `MaterializedMySQL` может авторизоваться только с помощью этого метода.
- `gtid_mode = on` — ведение журнала на основе GTID является обязательным для обеспечения правильной репликации.
:::note Внимание
При включении `gtid_mode` вы также должны указать `enforce_gtid_consistency = on`.
:::
## Виртуальные столбцы {#virtual-columns}
При работе с движком баз данных `MaterializedMySQL` используются таблицы семейства [ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) с виртуальными столбцами `_sign` и `_version`.
- `_version` — счетчик транзакций. Тип [UInt64](../../sql-reference/data-types/int-uint.md).
- `_sign` — метка удаления. Тип [Int8](../../sql-reference/data-types/int-uint.md). Возможные значения:
- `1` — строка не удалена,
- `-1` — строка удалена.
## Поддержка типов данных {#data_types-support}
| MySQL | ClickHouse |
|-------------------------|--------------------------------------------------------------|
| TINY | [Int8](../../sql-reference/data-types/int-uint.md) |
| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) |
| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) |
| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) |
| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) |
| FLOAT | [Float32](../../sql-reference/data-types/float.md) |
| DOUBLE | [Float64](../../sql-reference/data-types/float.md) |
| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) |
| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) |
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) |
| ENUM | [Enum](../../sql-reference/data-types/enum.md) |
| STRING | [String](../../sql-reference/data-types/string.md) |
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
| BLOB | [String](../../sql-reference/data-types/string.md) |
| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) |
Тип [Nullable](../../sql-reference/data-types/nullable.md) поддерживается.
Другие типы не поддерживаются. Если таблица MySQL содержит столбец другого типа, ClickHouse выдаст исключение "Неподдерживаемый тип данных" ("Unhandled data type") и остановит репликацию.
## Особенности и рекомендации {#specifics-and-recommendations}
### Ограничения совместимости {#compatibility-restrictions}
Кроме ограничений на типы данных, существует несколько ограничений по сравнению с базами данных MySQL, которые следует решить до того, как станет возможной репликация:
- Каждая таблица в MySQL должна содержать `PRIMARY KEY`.
- Репликация для таблиц, содержащих строки со значениями полей `ENUM` вне диапазона значений (определяется размерностью `ENUM`), не будет работать.
### DDL-запросы {#ddl-queries}
DDL-запросы в MySQL конвертируются в соответствующие DDL-запросы в ClickHouse ([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md)). Если ClickHouse не может конвертировать какой-либо DDL-запрос, он его игнорирует.
### Репликация данных {#data-replication}
Данные являются неизменяемыми со стороны пользователя ClickHouse, но автоматически обновляются путём репликации следующих запросов из MySQL:
- Запрос `INSERT` конвертируется в ClickHouse в `INSERT` с `_sign=1`.
- Запрос `DELETE` конвертируется в ClickHouse в `INSERT` с `_sign=-1`.
- Запрос `UPDATE` конвертируется в ClickHouse в `INSERT` с `_sign=-1` и `INSERT` с `_sign=1`.
### Выборка из таблиц движка MaterializedMySQL {#select}
Запрос `SELECT` из таблиц движка `MaterializedMySQL` имеет некоторую специфику:
- Если в запросе `SELECT` напрямую не указан столбец `_version`, то используется модификатор [FINAL](../../sql-reference/statements/select/from.md#select-from-final). Таким образом, выбираются только строки с `MAX(_version)`.
- Если в запросе `SELECT` напрямую не указан столбец `_sign`, то по умолчанию используется `WHERE _sign=1`. Таким образом, удаленные строки не включаются в результирующий набор.
- Результат включает комментарии к столбцам, если они существуют в таблицах базы данных MySQL.
### Конвертация индексов {#index-conversion}
Секции `PRIMARY KEY` и `INDEX` в MySQL конвертируются в кортежи `ORDER BY` в таблицах ClickHouse.
В таблицах ClickHouse данные физически хранятся в том порядке, который определяется секцией `ORDER BY`. Чтобы физически перегруппировать данные, используйте [материализованные представления](../../sql-reference/statements/create/view.md#materialized).
**Примечание**
- Строки с `_sign=-1` физически не удаляются из таблиц.
- Каскадные запросы `UPDATE/DELETE` не поддерживаются движком `MaterializedMySQL`.
- Репликация может быть легко нарушена.
- Прямые операции изменения данных в таблицах и базах данных `MaterializedMySQL` запрещены.
- На работу `MaterializedMySQL` влияет настройка [optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert). Когда таблица на MySQL сервере меняется, происходит слияние данных в соответсвующей таблице в базе данных `MaterializedMySQL`.
## Примеры использования {#examples-of-use}
Запросы в MySQL:
``` sql
mysql> CREATE DATABASE db;
mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
mysql> DELETE FROM db.test WHERE a=1;
mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
mysql> UPDATE db.test SET c='Wow!', b=222;
mysql> SELECT * FROM test;
```
```text
+---+------+------+
| a | b | c |
+---+------+------+
| 2 | 222 | Wow! |
+---+------+------+
```
База данных в ClickHouse, обмен данными с сервером MySQL:
База данных и созданная таблица:
``` sql
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
SHOW TABLES FROM mysql;
```
``` text
┌─name─┐
│ test │
└──────┘
```
После вставки данных:
``` sql
SELECT * FROM mysql.test;
```
``` text
┌─a─┬──b─┐
│ 1 │ 11 │
│ 2 │ 22 │
└───┴────┘
```
После удаления данных, добавления столбца и обновления:
``` sql
SELECT * FROM mysql.test;
```
``` text
┌─a─┬───b─┬─c────┐
│ 2 │ 222 │ Wow! │
└───┴─────┴──────┘
```

2
docs/ru/introduction/distinctive-features.md
View File

@ -33,7 +33,7 @@ sidebar_label: "Отличительные возможности ClickHouse"
## Поддержка SQL {#sql-support}
ClickHouse поддерживает [декларативный язык запросов на основе SQL](../sql-reference/index.md) и во [многих случаях](../sql-reference/ansi.mdx) совпадающий с SQL-стандартом.
ClickHouse поддерживает декларативный язык запросов SQL.
Поддерживаются [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), подзапросы в секциях [FROM](../sql-reference/statements/select/from.md), [IN](../sql-reference/operators/in.md), [JOIN](../sql-reference/statements/select/join.md), [функции window](../sql-reference/window-functions/index.mdx), а также скалярные подзапросы.

2
docs/ru/operations/settings/settings-users.md
View File

@ -30,7 +30,7 @@ sidebar_label: "Настройки пользователей"
<profile>profile_name</profile>
<quota>default</quota>
<default_database>default<default_database>
<default_database>default</default_database>
<databases>
<database_name>
<table_name>

2
docs/ru/sql-reference/aggregate-functions/reference/simplelinearregression.md
View File

@ -18,7 +18,7 @@ simpleLinearRegression(x, y)
Возвращаемые значения:
Константы `(a, b)` результирующей прямой `y = a*x + b`.
Константы `(k, b)` результирующей прямой `y = k*x + b`.
**Примеры**

10
docs/ru/sql-reference/ansi.mdx
View File

@ -1,10 +0,0 @@
---
slug: /ru/sql-reference/ansi
sidebar_position: 40
sidebar_label: ANSI Compatibility
title: "ANSI Compatibility"
---
import Content from '@site/docs/en/sql-reference/ansi.md';
<Content />

4
docs/ru/sql-reference/table-functions/generate.md
View File

@ -18,9 +18,9 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri
- `name` — название соответствующего столбца.
- `TypeName` — тип соответствующего столбца.
- `max_array_length` — максимальная длина массива для всех сгенерированных массивов. По умолчанию `10`.
- `max_string_length` — максимальная длина строки для всех генерируемых строк. По умолчанию `10`.
- `random_seed` — укажите состояние генератора случайных чисел вручную, чтобы получить стабильные результаты. Если значение равно `NULL` - генератор инициализируется случайным состоянием.
- `max_string_length` — максимальная длина строки для всех генерируемых строк. По умолчанию `10`.
- `max_array_length` — максимальная длина массива для всех сгенерированных массивов. По умолчанию `10`.
**Возвращаемое значение**

2
docs/zh/introduction/distinctive-features.md
View File

@ -37,7 +37,7 @@ ClickHouse会使用服务器上一切可用的资源，从而以最自然的方
## 支持SQL {#zhi-chi-sql}
ClickHouse支持一种[基于SQL的声明式查询语言](../sql-reference/index.md)，它在许多情况下与[ANSI SQL标准](../sql-reference/ansi.md)相同。
ClickHouse支持一种基于SQL的声明式查询语言。
支持的查询[GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md), [IN](../sql-reference/operators/in.md)以及非相关子查询。

2
docs/zh/sql-reference/aggregate-functions/reference/simplelinearregression.md
View File

@ -20,7 +20,7 @@ simpleLinearRegression(x, y)
**返回值**
符合`y = a*x + b`的常量 `(a, b)` 。
符合`y = k*x + b`的常量 `(k, b)` 。
**示例**

191
docs/zh/sql-reference/ansi.md
View File

@ -1,191 +0,0 @@
---
slug: /zh/sql-reference/ansi
sidebar_position: 40
sidebar_label: "ANSI\u517C\u5BB9\u6027"
---
# ClickHouse SQL方言 与ANSI SQL的兼容性{#ansi-sql-compatibility-of-clickhouse-sql-dialect}
:::note
本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准.
:::
## 行为差异 {#differences-in-behaviour}
下表列出了ClickHouse能够使用，但与ANSI SQL规定有差异的查询特性。
| 功能ID | 功能名称 | 差异 |
| ------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| E011 | 数值型数据类型 | 带小数点的数字被视为近似值 (`Float64`）而不是精确值 (`Decimal`) |
| E051-05 | SELECT 的列可以重命名 | 字段重命名的作用范围不限于进行重命名的SELECT子查询（参考[表达式别名](https://clickhouse.com/docs/zh/sql-reference/syntax/#notes-on-usage)） |
| E141-01 | NOT NULL（非空）约束 | ClickHouse表中每一列默认为`NOT NULL` |
| E011-04 | 算术运算符 | ClickHouse在运算时会进行溢出，而不是四舍五入。此外会根据自定义规则修改结果数据类型（参考[溢出检查](https://clickhouse.com/docs/zh/sql-reference/data-types/decimal/#yi-chu-jian-cha)） |
## 功能状态 {#feature-status}
| 功能ID | 功能名称 | 状态 | 注释 |
| -------- | ---------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| **E011** | **数值型数据类型** | **部分**{.text-warning} | |
| E011-01 | INTEGER （整型）和SMALLINT （小整型）数据类型 | 是 {.text-success} | |
| E011-02 | REAL （实数）、DOUBLE PRECISION （双精度浮点数）和FLOAT（单精度浮点数）数据类型数据类型 | 是 {.text-success} | |
| E011-03 | DECIMAL （精确数字）和NUMERIC （精确数字）数据类型 | 是 {.text-success} | |
| E011-04 | 算术运算符 | 是 {.text-success} | |
| E011-05 | 数值比较 | 是 {.text-success} | |
| E011-06 | 数值数据类型之间的隐式转换 | 否 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse针对不同数据类型有对应的比较函数和类型转换函数 |
| **E021** | **字符串类型** | **部分**{.text-warning} | |
| E021-01 | CHARACTER （字符串）数据类型 | 是 {.text-success} | |
| E021-02 | CHARACTER VARYING （可变字符串）数据类型 | 是 {.text-success} | |
| E021-03 | 字符字面量 | 是 {.text-success} | |
| E021-04 | CHARACTER_LENGTH 函数 | 部分 {.text-warning} | 不支持 `using` 从句 |
| E021-05 | OCTET_LENGTH 函数 | 否 {.text-danger} | 使用 `LENGTH` 函数代替 |
| E021-06 | SUBSTRING | 部分 {.text-warning} | 不支持 `SIMILAR` 和 `ESCAPE` 从句，没有`SUBSTRING_REGEX` 函数 |
| E021-07 | 字符串拼接 | 部分 {.text-warning} | 不支持 `COLLATE` 从句 |
| E021-08 | 大小写转换 | 是 {.text-success} | |
| E021-09 | 裁剪字符串 | 是 {.text-success} | |
| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 部分 {.text-warning} | ANSI SQL允许在数据类型之间进行任意隐式转换，而ClickHouse针对不同数据类型有对应的比较函数和类型转换函数 |
| E021-11 | POSITION 函数 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 从句，不支持`POSITION_REGEX`函数 |
| E021-12 | 字符串比较 | 是 {.text-success} | |
| **E031** | **标识符** | **部分**{.text-warning} | |
| E031-01 | 分隔标识符 | 部分 {.text-warning} | Unicode文字支持有限 |
| E031-02 | 小写标识符 | 是 {.text-success} | |
| E031-03 | 标识符最后加下划线 | 是 {.text-success} | |
| **E051** | **基本查询规范** | **部分**{.text-warning} | |
| E051-01 | SELECT DISTINCT | 是 {.text-success} | |
| E051-02 | GROUP BY 从句 | 是 {.text-success} | |
| E051-04 | GROUP BY 从句中的列可以包含不在 `<select list>`中出现的列 | 是 {.text-success} | |
| E051-05 | SELECT 的列可以重命名 | 是 {.text-success} | |
| E051-06 | HAVING 从句 | 是 {.text-success} | |
| E051-07 | SELECT 选择的列中允许出现\* | 是 {.text-success} | |
| E051-08 | FROM 从句中的关联名称 | 是 {.text-success} | |
| E051-09 | 重命名 FROM 从句中的列 | 否 {.text-danger} | |
| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | |
| E061-01 | 比较谓词 | 是 {.text-success} | |
| E061-02 | BETWEEN 谓词 | 部分 {.text-warning} | 不支持 `SYMMETRIC` 和 `ASYMMETRIC` 从句 |
| E061-03 | IN 谓词后可接值列表 | 是 {.text-success} | |
| E061-04 | LIKE 谓词 | 是 {.text-success} | |
| E061-05 | LIKE 谓词后接 ESCAPE 从句 | 否 {.text-danger} | |
| E061-06 | NULL 谓词 | 是 {.text-success} | |
| E061-07 | 量化比较谓词（ALL、SOME、ANY） | 否 {.text-danger} | |
| E061-08 | EXISTS 谓词 | 否 {.text-danger} | |
| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | |
| E061-11 | IN 谓词中的子查询 | 是 {.text-success} | |
| E061-12 | 量化比较谓词（BETWEEN、IN、LIKE）中的子查询 | 否 {.text-danger} | |
| E061-13 | 相关子查询 | 否 {.text-danger} | |
| E061-14 | 搜索条件 | 是 {.text-success} | |
| **E071** | **基本查询表达式** | **部分**{.text-warning} | |
| E071-01 | UNION DISTINCT 表运算符 | 是 {.text-success} | |
| E071-02 | UNION ALL 表运算符 | 是 {.text-success} | |
| E071-03 | EXCEPT DISTINCT 表运算符 | 否 {.text-danger} | |
| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | |
| E071-06 | 子查询中的表运算符 | 是 {.text-success} | |
| **E081** | **基本权限** | **是**{.text-success} | |
| E081-01 | 表级别的SELECT（查询）权限 | 是 {.text-success} | |
| E081-02 | DELETE（删除）权限 | 是 {.text-success} | |
| E081-03 | 表级别的INSERT（插入）权限 | 是 {.text-success} | |
| E081-04 | 表级别的UPDATE（更新）权限 | 是 {.text-success} | |
| E081-05 | 列级别的UPDATE（更新）权限 | 是 {.text-success} | |
| E081-06 | 表级别的REFERENCES（引用）权限 | 是 {.text-success} | |
| E081-07 | 列级别的REFERENCES（引用）权限 | 是 {.text-success} | |
| E081-08 | WITH GRANT OPTION | 是 {.text-success} | |
| E081-09 | USAGE（使用）权限 | 是 {.text-success} | |
| E081-10 | EXECUTE（执行）权限 | 是 {.text-success} | |
| **E091** | **集合函数** | **是**{.text-success} | |
| E091-01 | AVG | 是 {.text-success} | |
| E091-02 | COUNT | 是 {.text-success} | |
| E091-03 | MAX | 是 {.text-success} | |
| E091-04 | MIN | 是 {.text-success} | |
| E091-05 | SUM | 是 {.text-success} | |
| E091-06 | ALL修饰词 | 否。 {.text-danger} | |
| E091-07 | DISTINCT修饰词 | 是 {.text-success} | 并非所有聚合函数都支持该修饰词 |
| **E101** | **基本数据操作** | **部分**{.text-warning} | |
| E101-01 | INSERT（插入）语句 | 是 {.text-success} | 注：ClickHouse中的主键并不隐含`UNIQUE` 约束 |
| E101-03 | 可指定范围的UPDATE（更新）语句 | 部分 {.text-warning} | `ALTER UPDATE` 语句用来批量更新数据 |
| E101-04 | 可指定范围的DELETE（删除）语句 | 部分 {.text-warning} | `ALTER DELETE` 语句用来批量删除数据 |
| **E111** | **返回一行的SELECT语句** | **否**{.text-danger} | |
| **E121** | **基本游标支持** | **否**{.text-danger} | |
| E121-01 | DECLARE CURSOR | 否 {.text-danger} | |
| E121-02 | ORDER BY 涉及的列不需要出现在SELECT的列中 | 是 {.text-success} | |
| E121-03 | ORDER BY 从句中的表达式 | 是 {.text-success} | |
| E121-04 | OPEN 语句 | 否 {.text-danger} | |
| E121-06 | 受游标位置控制的 UPDATE 语句 | 否 {.text-danger} | |
| E121-07 | 受游标位置控制的 DELETE 语句 | 否 {.text-danger} | |
| E121-08 | CLOSE 语句 | 否 {.text-danger} | |
| E121-10 | FETCH 语句中包含隐式NEXT | 否 {.text-danger} | |
| E121-17 | WITH HOLD 游标 | 否 {.text-danger} | |
| **E131** | **空值支持** | **是**{.text-success} | 有部分限制 |
| **E141** | **基本完整性约束** | **部分**{.text-warning} | |
| E141-01 | NOT NULL（非空）约束 | 是 {.text-success} | 注: 默认情况下ClickHouse表中的列隐含`NOT NULL`约束 |
| E141-02 | NOT NULL（非空）列的UNIQUE（唯一）约束 | 否 {.text-danger} | |
| E141-03 | 主键约束 | 部分 {.text-warning} | |
| E141-04 | 对于引用删除和引用更新操作，基本的FOREIGN KEY（外键）约束默认不进行任何操作（NO ACTION） | 否 {.text-danger} | |
| E141-06 | CHECK（检查）约束 | 是 {.text-success} | |
| E141-07 | 列默认值 | 是 {.text-success} | |
| E141-08 | 在主键上推断非空 | 是 {.text-success} | |
| E141-10 | 可以按任何顺序指定外键中的名称 | 否 {.text-danger} | |
| **E151** | **事务支持** | **否**{.text-danger} | |
| E151-01 | COMMIT（提交）语句 | 否 {.text-danger} | |
| E151-02 | ROLLBACK（回滚）语句 | 否 {.text-danger} | |
| **E152** | **基本的SET TRANSACTION（设置事务隔离级别）语句** | **否**{.text-danger} | |
| E152-01 | SET TRANSACTION语句：ISOLATION LEVEL SERIALIZABLE（隔离级别为串行化）从句 | 否 {.text-danger} | |
| E152-02 | SET TRANSACTION语句：READ ONLY（只读）和READ WRITE（读写）从句 | 否 {.text-danger} | |
| **E153** | **具有子查询的可更新查询** | **是**{.text-success} | |
| **E161** | **使用“--”符号作为SQL注释** | **是**{.text-success} | |
| **E171** | **SQLSTATE支持** | **否**{.text-danger} | |
| **E182** | **主机语言绑定** | **否**{.text-danger} | |
| **F031** | **基本架构操作** | **部分**{.text-warning} | |
| F031-01 | 使用 CREATE TABLE 语句创建持久表 | 部分 {.text-warning} | 不支持 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 从句，不支持用户解析的数据类型 |
| F031-02 | CREATE VIEW（创建视图）语句 | 部分 {.text-warning} | 不支持 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 从句，不支持用户解析的数据类型 |
| F031-03 | GRANT（授权）语句 | 是 {.text-success} | |
| F031-04 | ALTER TABLE语句：ADD COLUMN从句 | 是 {.text-success} | 不支持 `GENERATED` 从句和以系统时间做参数 |
| F031-13 | DROP TABLE语句：RESTRICT从句 | 否 {.text-danger} | |
| F031-16 | DROP VIEW语句：RESTRICT子句 | 否 {.text-danger} | |
| F031-19 | REVOKE语句：RESTRICT子句 | 否 {.text-danger} | |
| **F041** | **基本连接关系** | **部分**{.text-warning} | |
| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | |
| F041-02 | INNER 关键字 | 是 {.text-success} | |
| F041-03 | LEFT OUTER JOIN | 是 {.text-success} | |
| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | |
| F041-05 | 外连接可嵌套 | 是 {.text-success} | |
| F041-07 | 左外部连接或右外连接中的内部表也可用于内部联接 | 是 {.text-success} | |
| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否 {.text-danger} | |
| **F051** | **基本日期和时间** | **部分**{.text-warning} | |
| F051-01 | DATE（日期）数据类型（并支持用于表达日期的字面量) | 是 {.text-success} | |
| F051-02 | TIME（时间）数据类型（并支持用于表达时间的字面量），小数秒精度至少为0 | 否 {.text-danger} | |
| F051-03 | 时间戳数据类型（并支持用于表达时间戳的字面量），小数秒精度至少为0和6 | 是 {.text-danger} | |
| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 是 {.text-success} | |
| F051-05 | DateTime 类型和字符串形式表达的时间之间的显式转换 | 是 {.text-success} | |
| F051-06 | CURRENT_DATE | 否 {.text-danger} | 使用`today()`替代 |
| F051-07 | LOCALTIME | 否 {.text-danger} | 使用`now()`替代 |
| F051-08 | LOCALTIMESTAMP | 否 {.text-danger} | |
| **F081** | **视图的UNION和EXCEPT操作** | **部分**{.text-warning} | |
| **F131** | **分组操作** | **部分**{.text-warning} | |
| F131-01 | 在具有分组视图的查询中支持 WHERE、GROUP BY 和 HAVING 子句 | 是 {.text-success} | |
| F131-02 | 在分组视图中支持多张表 | 是 {.text-success} | |
| F131-03 | 分组视图的查询中支持集合函数 | 是 {.text-success} | |
| F131-04 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的子查询 | 是 {.text-success} | |
| F131-05 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的仅返回1条记录的SELECT查询 | 否 {.text-danger} | |
| **F181** | **多模块支持** | **否**{.text-danger} | |
| **F201** | **CAST 函数** | **是**{.text-success} | |
| **F221** | **显式默认值** | **否**{.text-danger} | |
| **F261** | **CASE 表达式** | **是**{.text-success} | |
| F261-01 | 简单 CASE 表达式 | 是 {.text-success} | |
| F261-02 | 搜索型 CASE 表达式 | 是 {.text-success} | |
| F261-03 | NULLIF | 是 {.text-success} | |
| F261-04 | COALESCE | 是 {.text-success} | |
| **F311** | **架构定义语句** | **部分**{.text-warning} | |
| F311-01 | CREATE SCHEMA | 部分 {.text-warning} | 见`CREATE DATABASE` |
| F311-02 | 用于创建持久表的 CREATE TABLE | 是 {.text-success} | |
| F311-03 | CREATE VIEW | 是 {.text-success} | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否 {.text-danger} | |
| F311-05 | GRANT 语句 | 是 {.text-success} | |
| **F471** | **标量子查询** | **是**{.text-success} | |
| **F481** | **扩展 NULL 谓词** | **是**{.text-success} | |
| **F812** | **基本标志位** | **否**{.text-danger} |
| **S011** | **用于不重复数据的数据类型** | **否**{.text-danger} |
| **T321** | **基本的SQL调用例程** | **否**{.text-danger} | |
| T321-01 | 没有重载的用户定义函数 | 否{.text-danger} | |
| T321-02 | 没有重载的用户定义存储过程 | 否{.text-danger} | |
| T321-03 | 功能调用 | 否 {.text-danger} | |
| T321-04 | CALL 语句 | 否 {.text-danger} | |
| T321-05 | RETURN 语句 | 否 {.text-danger} | |
| **T631** | **IN 谓词后接一个列表** | **是**{.text-success} | |

4
docs/zh/sql-reference/table-functions/generate.md
View File

@ -18,9 +18,9 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri
- `name` — 对应列的名称。
- `TypeName` — 对应列的类型。
- `max_array_length` — 生成数组的最大长度。 默认为10。
- `max_string_length` — 生成字符串的最大长度。 默认为10。
- `random_seed` — 手动指定随机种子以产生稳定的结果。 如果为NULL-种子是随机生成的。
- `max_string_length` — 生成字符串的最大长度。 默认为10。
- `max_array_length` — 生成数组的最大长度。 默认为10。
**返回值**

2
programs/client/Client.cpp
View File

@ -512,6 +512,7 @@ void Client::connect()
{
std::cout << "Connected to " << server_name << " server version " << server_version << "." << std::endl << std::endl;
#ifndef CLICKHOUSE_CLOUD
auto client_version_tuple = std::make_tuple(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH);
auto server_version_tuple = std::make_tuple(server_version_major, server_version_minor, server_version_patch);
@ -527,6 +528,7 @@ void Client::connect()
<< "It may indicate that the server is out of date and can be upgraded." << std::endl
<< std::endl;
}
#endif
}
if (!client_context->getSettingsRef()[Setting::use_client_time_zone])

26
programs/disks/CommandCopy.cpp
View File

@ -38,18 +38,10 @@ public:
String path_to = disk_to.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
bool recursive = options.count("recursive");
if (!disk_from.getDisk()->exists(path_from))
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"cannot stat '{}' on disk '{}': No such file or directory",
path_from,
disk_from.getDisk()->getName());
}
if (disk_from.getDisk()->isFile(path_from))
if (disk_from.getDisk()->existsFile(path_from))
{
auto target_location = getTargetLocation(path_from, disk_to, path_to);
if (!disk_to.getDisk()->exists(target_location) || disk_to.getDisk()->isFile(target_location))
if (!disk_to.getDisk()->existsDirectory(target_location))
{
disk_from.getDisk()->copyFile(
path_from,
@ -65,7 +57,7 @@ public:
ErrorCodes::BAD_ARGUMENTS, "cannot overwrite directory {} with non-directory {}", target_location, path_from);
}
}
else if (disk_from.getDisk()->isDirectory(path_from))
else if (disk_from.getDisk()->existsDirectory(path_from))
{
if (!recursive)
{
@ -73,11 +65,11 @@ public:
}
auto target_location = getTargetLocation(path_from, disk_to, path_to);
if (disk_to.getDisk()->isFile(target_location))
if (disk_to.getDisk()->existsFile(target_location))
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory {} with directory {}", path_to, target_location);
}
if (!disk_to.getDisk()->exists(target_location))
if (!disk_to.getDisk()->existsDirectory(target_location))
{
disk_to.getDisk()->createDirectory(target_location);
}
@ -89,6 +81,14 @@ public:
/* write_settings= */ {},
/* cancellation_hook= */ {});
}
else
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"cannot stat '{}' on disk '{}': No such file or directory",
path_from,
disk_from.getDisk()->getName());
}
}
};

10
programs/disks/CommandMove.cpp
View File

@ -30,21 +30,21 @@ public:
String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
String path_to = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
if (disk.getDisk()->isFile(path_from))
if (disk.getDisk()->existsFile(path_from))
{
disk.getDisk()->moveFile(path_from, path_to);
}
else if (disk.getDisk()->isDirectory(path_from))
else if (disk.getDisk()->existsDirectory(path_from))
{
auto target_location = getTargetLocation(path_from, disk, path_to);
if (!disk.getDisk()->exists(target_location))
if (!disk.getDisk()->existsDirectory(target_location))
{
disk.getDisk()->createDirectory(target_location);
disk.getDisk()->moveDirectory(path_from, target_location);
}
else
{
if (disk.getDisk()->isFile(target_location))
if (disk.getDisk()->existsFile(target_location))
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory '{}' with directory '{}'", target_location, path_from);
@ -57,7 +57,7 @@ public:
disk.getDisk()->moveDirectory(path_from, target_location);
}
}
else if (!disk.getDisk()->exists(path_from))
else
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,

10
programs/disks/CommandRemove.cpp
View File

@ -28,11 +28,7 @@ public:
auto disk = client.getCurrentDiskWithPath();
const String & path = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path"));
bool recursive = options.count("recursive");
if (!disk.getDisk()->exists(path))
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} on disk {} doesn't exist", path, disk.getDisk()->getName());
}
if (disk.getDisk()->isDirectory(path))
if (disk.getDisk()->existsDirectory(path))
{
if (!recursive)
{
@ -41,10 +37,12 @@ public:
disk.getDisk()->removeRecursive(path);
}
else
else if (disk.getDisk()->existsFile(path))
{
disk.getDisk()->removeFileIfExists(path);
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} on disk {} doesn't exist", path, disk.getDisk()->getName());
}
};

7
programs/disks/CommandWrite.cpp
View File

@ -1,6 +1,7 @@
#include <Interpreters/Context.h>
#include "ICommand.h"
#include <IO/ReadBufferFromEmptyFile.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/copyData.h>
@ -36,7 +37,11 @@ public:
return std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
String relative_path_from = disk.getRelativeFromRoot(path_from.value());
return disk.getDisk()->readFile(relative_path_from, getReadSettings());
auto res = disk.getDisk()->readFileIfExists(relative_path_from, getReadSettings());
if (res)
return res;
/// For backward compatibility.
return std::make_unique<ReadBufferFromEmptyFile>();
}();
auto out = disk.getDisk()->writeFile(path_to);

2
programs/disks/DisksClient.cpp
View File

@ -29,7 +29,7 @@ DiskWithPath::DiskWithPath(DiskPtr disk_, std::optional<String> path_) : disk(di
}
String relative_path = normalizePathAndGetAsRelative(path);
if (disk->isDirectory(relative_path) || (relative_path.empty() && (disk->isDirectory("/"))))
if (disk->existsDirectory(relative_path) || (relative_path.empty() && (disk->existsDirectory("/"))))
{
return;
}

2
programs/disks/DisksClient.h
View File

@ -33,7 +33,7 @@ public:
bool isDirectory(const String & any_path) const
{
return disk->isDirectory(getRelativeFromRoot(any_path)) || (getRelativeFromRoot(any_path).empty() && (disk->isDirectory("/")));
return disk->existsDirectory(getRelativeFromRoot(any_path)) || (getRelativeFromRoot(any_path).empty() && (disk->existsDirectory("/")));
}
std::vector<String> listAllFilesByPath(const String & any_path) const;

2
programs/disks/ICommand.h
View File

@ -90,7 +90,7 @@ protected:
String getTargetLocation(const String & path_from, DiskWithPath & disk_to, const String & path_to)
{
if (!disk_to.getDisk()->isDirectory(path_to))
if (!disk_to.getDisk()->existsDirectory(path_to))
{
return path_to;
}

3
programs/library-bridge/LibraryBridgeHandlers.cpp
View File

@ -594,7 +594,10 @@ void CatBoostLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & requ
catch (...)
{
tryLogCurrentException(log);
out.cancel();
}
return;
}
try

120
programs/local/LocalServer.cpp
View File

@ -77,6 +77,42 @@ namespace Setting
extern const SettingsLocalFSReadMethod storage_file_read_method;
}
namespace ServerSetting
{
extern const ServerSettingsDouble cache_size_to_ram_max_ratio;
extern const ServerSettingsUInt64 compiled_expression_cache_elements_size;
extern const ServerSettingsUInt64 compiled_expression_cache_size;
extern const ServerSettingsUInt64 database_catalog_drop_table_concurrency;
extern const ServerSettingsString default_database;
extern const ServerSettingsString index_mark_cache_policy;
extern const ServerSettingsUInt64 index_mark_cache_size;
extern const ServerSettingsDouble index_mark_cache_size_ratio;
extern const ServerSettingsString index_uncompressed_cache_policy;
extern const ServerSettingsUInt64 index_uncompressed_cache_size;
extern const ServerSettingsDouble index_uncompressed_cache_size_ratio;
extern const ServerSettingsUInt64 io_thread_pool_queue_size;
extern const ServerSettingsString mark_cache_policy;
extern const ServerSettingsUInt64 mark_cache_size;
extern const ServerSettingsDouble mark_cache_size_ratio;
extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size;
extern const ServerSettingsUInt64 max_io_thread_pool_free_size;
extern const ServerSettingsUInt64 max_io_thread_pool_size;
extern const ServerSettingsUInt64 max_outdated_parts_loading_thread_pool_size;
extern const ServerSettingsUInt64 max_parts_cleaning_thread_pool_size;
extern const ServerSettingsUInt64 max_server_memory_usage;
extern const ServerSettingsDouble max_server_memory_usage_to_ram_ratio;
extern const ServerSettingsUInt64 max_thread_pool_free_size;
extern const ServerSettingsUInt64 max_thread_pool_size;
extern const ServerSettingsUInt64 max_unexpected_parts_loading_thread_pool_size;
extern const ServerSettingsUInt64 mmap_cache_size;
extern const ServerSettingsBool show_addresses_in_stack_traces;
extern const ServerSettingsUInt64 thread_pool_queue_size;
extern const ServerSettingsString uncompressed_cache_policy;
extern const ServerSettingsUInt64 uncompressed_cache_size;
extern const ServerSettingsDouble uncompressed_cache_size_ratio;
extern const ServerSettingsBool use_legacy_mongodb_integration;
}
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
@ -157,9 +193,9 @@ void LocalServer::initialize(Poco::Util::Application & self)
server_settings.loadSettingsFromConfig(config());
GlobalThreadPool::initialize(
server_settings.max_thread_pool_size,
server_settings.max_thread_pool_free_size,
server_settings.thread_pool_queue_size);
server_settings[ServerSetting::max_thread_pool_size],
server_settings[ServerSetting::max_thread_pool_free_size],
server_settings[ServerSetting::thread_pool_queue_size]);
#if USE_AZURE_BLOB_STORAGE
/// See the explanation near the same line in Server.cpp
@ -170,17 +206,17 @@ void LocalServer::initialize(Poco::Util::Application & self)
#endif
getIOThreadPool().initialize(
server_settings.max_io_thread_pool_size,
server_settings.max_io_thread_pool_free_size,
server_settings.io_thread_pool_queue_size);
server_settings[ServerSetting::max_io_thread_pool_size],
server_settings[ServerSetting::max_io_thread_pool_free_size],
server_settings[ServerSetting::io_thread_pool_queue_size]);
const size_t active_parts_loading_threads = server_settings.max_active_parts_loading_thread_pool_size;
const size_t active_parts_loading_threads = server_settings[ServerSetting::max_active_parts_loading_thread_pool_size];
getActivePartsLoadingThreadPool().initialize(
active_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded
active_parts_loading_threads);
const size_t outdated_parts_loading_threads = server_settings.max_outdated_parts_loading_thread_pool_size;
const size_t outdated_parts_loading_threads = server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size];
getOutdatedPartsLoadingThreadPool().initialize(
outdated_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded
@ -188,7 +224,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
const size_t unexpected_parts_loading_threads = server_settings.max_unexpected_parts_loading_thread_pool_size;
const size_t unexpected_parts_loading_threads = server_settings[ServerSetting::max_unexpected_parts_loading_thread_pool_size];
getUnexpectedPartsLoadingThreadPool().initialize(
unexpected_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded
@ -196,16 +232,16 @@ void LocalServer::initialize(Poco::Util::Application & self)
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
const size_t cleanup_threads = server_settings.max_parts_cleaning_thread_pool_size;
const size_t cleanup_threads = server_settings[ServerSetting::max_parts_cleaning_thread_pool_size];
getPartsCleaningThreadPool().initialize(
cleanup_threads,
0, // We don't need any threads one all the parts will be deleted
cleanup_threads);
getDatabaseCatalogDropTablesThreadPool().initialize(
server_settings.database_catalog_drop_table_concurrency,
server_settings[ServerSetting::database_catalog_drop_table_concurrency],
0, // We don't need any threads if there are no DROP queries.
server_settings.database_catalog_drop_table_concurrency);
server_settings[ServerSetting::database_catalog_drop_table_concurrency]);
}
@ -296,9 +332,14 @@ void LocalServer::tryInitPath()
global_context->setUserFilesPath(""); /// user's files are everywhere
std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts/");
std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts" / "");
global_context->setUserScriptsPath(user_scripts_path);
/// Set path for filesystem caches
String filesystem_caches_path(getClientConfiguration().getString("filesystem_caches_path", fs::path(path) / "cache" / ""));
if (!filesystem_caches_path.empty())
global_context->setFilesystemCachesPath(filesystem_caches_path);
/// top_level_domains_lists
const std::string & top_level_domains_path = getClientConfiguration().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
if (!top_level_domains_path.empty())
@ -470,7 +511,7 @@ try
UseSSL use_ssl;
thread_status.emplace();
StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces);
StackTrace::setShowAddresses(server_settings[ServerSetting::show_addresses_in_stack_traces]);
setupSignalHandler();
@ -507,10 +548,10 @@ try
/// Don't initialize DateLUT
registerFunctions();
registerAggregateFunctions();
registerTableFunctions(server_settings.use_legacy_mongodb_integration);
registerTableFunctions(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDatabases();
registerStorages(server_settings.use_legacy_mongodb_integration);
registerDictionaries(server_settings.use_legacy_mongodb_integration);
registerStorages(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDictionaries(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDisks(/* global_skip_access_check= */ true);
registerFormats();
@ -659,8 +700,8 @@ void LocalServer::processConfig()
const size_t physical_server_memory = getMemoryAmount();
size_t max_server_memory_usage = server_settings.max_server_memory_usage;
double max_server_memory_usage_to_ram_ratio = server_settings.max_server_memory_usage_to_ram_ratio;
size_t max_server_memory_usage = server_settings[ServerSetting::max_server_memory_usage];
double max_server_memory_usage_to_ram_ratio = server_settings[ServerSetting::max_server_memory_usage_to_ram_ratio];
size_t default_max_server_memory_usage = static_cast<size_t>(physical_server_memory * max_server_memory_usage_to_ram_ratio);
@ -689,12 +730,12 @@ void LocalServer::processConfig()
total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
const double cache_size_to_ram_max_ratio = server_settings.cache_size_to_ram_max_ratio;
const double cache_size_to_ram_max_ratio = server_settings[ServerSetting::cache_size_to_ram_max_ratio];
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * cache_size_to_ram_max_ratio);
String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
double uncompressed_cache_size_ratio = server_settings.uncompressed_cache_size_ratio;
String uncompressed_cache_policy = server_settings[ServerSetting::uncompressed_cache_policy];
size_t uncompressed_cache_size = server_settings[ServerSetting::uncompressed_cache_size];
double uncompressed_cache_size_ratio = server_settings[ServerSetting::uncompressed_cache_size_ratio];
if (uncompressed_cache_size > max_cache_size)
{
uncompressed_cache_size = max_cache_size;
@ -702,9 +743,9 @@ void LocalServer::processConfig()
}
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
String mark_cache_policy = server_settings.mark_cache_policy;
size_t mark_cache_size = server_settings.mark_cache_size;
double mark_cache_size_ratio = server_settings.mark_cache_size_ratio;
String mark_cache_policy = server_settings[ServerSetting::mark_cache_policy];
size_t mark_cache_size = server_settings[ServerSetting::mark_cache_size];
double mark_cache_size_ratio = server_settings[ServerSetting::mark_cache_size_ratio];
if (!mark_cache_size)
LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
if (mark_cache_size > max_cache_size)
@ -714,9 +755,9 @@ void LocalServer::processConfig()
}
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
String index_uncompressed_cache_policy = server_settings.index_uncompressed_cache_policy;
size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
double index_uncompressed_cache_size_ratio = server_settings.index_uncompressed_cache_size_ratio;
String index_uncompressed_cache_policy = server_settings[ServerSetting::index_uncompressed_cache_policy];
size_t index_uncompressed_cache_size = server_settings[ServerSetting::index_uncompressed_cache_size];
double index_uncompressed_cache_size_ratio = server_settings[ServerSetting::index_uncompressed_cache_size_ratio];
if (index_uncompressed_cache_size > max_cache_size)
{
index_uncompressed_cache_size = max_cache_size;
@ -724,9 +765,9 @@ void LocalServer::processConfig()
}
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
String index_mark_cache_policy = server_settings.index_mark_cache_policy;
size_t index_mark_cache_size = server_settings.index_mark_cache_size;
double index_mark_cache_size_ratio = server_settings.index_mark_cache_size_ratio;
String index_mark_cache_policy = server_settings[ServerSetting::index_mark_cache_policy];
size_t index_mark_cache_size = server_settings[ServerSetting::index_mark_cache_size];
double index_mark_cache_size_ratio = server_settings[ServerSetting::index_mark_cache_size_ratio];
if (index_mark_cache_size > max_cache_size)
{
index_mark_cache_size = max_cache_size;
@ -734,7 +775,7 @@ void LocalServer::processConfig()
}
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
size_t mmap_cache_size = server_settings.mmap_cache_size;
size_t mmap_cache_size = server_settings[ServerSetting::mmap_cache_size];
if (mmap_cache_size > max_cache_size)
{
mmap_cache_size = max_cache_size;
@ -746,8 +787,8 @@ void LocalServer::processConfig()
global_context->setQueryCache(0, 0, 0, 0);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size;
size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size;
size_t compiled_expression_cache_max_size_in_bytes = server_settings[ServerSetting::compiled_expression_cache_size];
size_t compiled_expression_cache_max_elements = server_settings[ServerSetting::compiled_expression_cache_elements_size];
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
@ -767,7 +808,7 @@ void LocalServer::processConfig()
/// We load temporary database first, because projections need it.
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
std::string default_database = server_settings.default_database;
std::string default_database = server_settings[ServerSetting::default_database];
DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
global_context->setCurrentDatabase(default_database);
@ -853,6 +894,7 @@ void LocalServer::addOptions(OptionsDescription & options_description)
{
options_description.main_description->add_options()
("table,N", po::value<std::string>(), "name of the initial table")
("copy", "shortcut for format conversion, equivalent to: --query 'SELECT * FROM table'")
/// If structure argument is omitted then initial query is not generated
("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")
@ -925,6 +967,12 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp
getClientConfiguration().setString("send_logs_level", options["send_logs_level"].as<std::string>());
if (options.count("wait_for_suggestions_to_load"))
getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
if (options.count("copy"))
{
if (!queries.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--copy' and '--query' cannot be specified at the same time");
queries.emplace_back("SELECT * FROM table");
}
}
void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector<Arguments> &, std::vector<Arguments> &)

435
programs/server/Server.cpp
View File

@ -164,6 +164,123 @@ namespace MergeTreeSetting
extern const MergeTreeSettingsBool allow_remote_fs_zero_copy_replication;
}
namespace ServerSetting
{
extern const ServerSettingsUInt32 asynchronous_heavy_metrics_update_period_s;
extern const ServerSettingsUInt32 asynchronous_metrics_update_period_s;
extern const ServerSettingsBool async_insert_queue_flush_on_shutdown;
extern const ServerSettingsUInt64 async_insert_threads;
extern const ServerSettingsBool async_load_databases;
extern const ServerSettingsUInt64 background_buffer_flush_schedule_pool_size;
extern const ServerSettingsUInt64 background_common_pool_size;
extern const ServerSettingsUInt64 background_distributed_schedule_pool_size;
extern const ServerSettingsUInt64 background_fetches_pool_size;
extern const ServerSettingsFloat background_merges_mutations_concurrency_ratio;
extern const ServerSettingsString background_merges_mutations_scheduling_policy;
extern const ServerSettingsUInt64 background_message_broker_schedule_pool_size;
extern const ServerSettingsUInt64 background_move_pool_size;
extern const ServerSettingsUInt64 background_pool_size;
extern const ServerSettingsUInt64 background_schedule_pool_size;
extern const ServerSettingsUInt64 backups_io_thread_pool_queue_size;
extern const ServerSettingsDouble cache_size_to_ram_max_ratio;
extern const ServerSettingsDouble cannot_allocate_thread_fault_injection_probability;
extern const ServerSettingsUInt64 cgroups_memory_usage_observer_wait_time;
extern const ServerSettingsUInt64 compiled_expression_cache_elements_size;
extern const ServerSettingsUInt64 compiled_expression_cache_size;
extern const ServerSettingsUInt64 concurrent_threads_soft_limit_num;
extern const ServerSettingsUInt64 concurrent_threads_soft_limit_ratio_to_cores;
extern const ServerSettingsUInt64 config_reload_interval_ms;
extern const ServerSettingsUInt64 database_catalog_drop_table_concurrency;
extern const ServerSettingsString default_database;
extern const ServerSettingsBool disable_internal_dns_cache;
extern const ServerSettingsUInt64 disk_connections_soft_limit;
extern const ServerSettingsUInt64 disk_connections_store_limit;
extern const ServerSettingsUInt64 disk_connections_warn_limit;
extern const ServerSettingsBool dns_allow_resolve_names_to_ipv4;
extern const ServerSettingsBool dns_allow_resolve_names_to_ipv6;
extern const ServerSettingsUInt64 dns_cache_max_entries;
extern const ServerSettingsInt32 dns_cache_update_period;
extern const ServerSettingsUInt32 dns_max_consecutive_failures;
extern const ServerSettingsBool enable_azure_sdk_logging;
extern const ServerSettingsBool format_alter_operations_with_parentheses;
extern const ServerSettingsUInt64 global_profiler_cpu_time_period_ns;
extern const ServerSettingsUInt64 global_profiler_real_time_period_ns;
extern const ServerSettingsDouble gwp_asan_force_sample_probability;
extern const ServerSettingsUInt64 http_connections_soft_limit;
extern const ServerSettingsUInt64 http_connections_store_limit;
extern const ServerSettingsUInt64 http_connections_warn_limit;
extern const ServerSettingsString index_mark_cache_policy;
extern const ServerSettingsUInt64 index_mark_cache_size;
extern const ServerSettingsDouble index_mark_cache_size_ratio;
extern const ServerSettingsString index_uncompressed_cache_policy;
extern const ServerSettingsUInt64 index_uncompressed_cache_size;
extern const ServerSettingsDouble index_uncompressed_cache_size_ratio;
extern const ServerSettingsUInt64 io_thread_pool_queue_size;
extern const ServerSettingsSeconds keep_alive_timeout;
extern const ServerSettingsString mark_cache_policy;
extern const ServerSettingsUInt64 mark_cache_size;
extern const ServerSettingsDouble mark_cache_size_ratio;
extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size;
extern const ServerSettingsUInt64 max_backups_io_thread_pool_free_size;
extern const ServerSettingsUInt64 max_backups_io_thread_pool_size;
extern const ServerSettingsUInt64 max_concurrent_insert_queries;
extern const ServerSettingsUInt64 max_concurrent_queries;
extern const ServerSettingsUInt64 max_concurrent_select_queries;
extern const ServerSettingsInt32 max_connections;
extern const ServerSettingsUInt64 max_database_num_to_warn;
extern const ServerSettingsUInt32 max_database_replicated_create_table_thread_pool_size;
extern const ServerSettingsUInt64 max_dictionary_num_to_warn;
extern const ServerSettingsUInt64 max_io_thread_pool_free_size;
extern const ServerSettingsUInt64 max_io_thread_pool_size;
extern const ServerSettingsUInt64 max_keep_alive_requests;
extern const ServerSettingsUInt64 max_outdated_parts_loading_thread_pool_size;
extern const ServerSettingsUInt64 max_partition_size_to_drop;
extern const ServerSettingsUInt64 max_part_num_to_warn;
extern const ServerSettingsUInt64 max_parts_cleaning_thread_pool_size;
extern const ServerSettingsUInt64 max_server_memory_usage;
extern const ServerSettingsDouble max_server_memory_usage_to_ram_ratio;
extern const ServerSettingsUInt64 max_table_num_to_warn;
extern const ServerSettingsUInt64 max_table_size_to_drop;
extern const ServerSettingsUInt64 max_temporary_data_on_disk_size;
extern const ServerSettingsUInt64 max_thread_pool_free_size;
extern const ServerSettingsUInt64 max_thread_pool_size;
extern const ServerSettingsUInt64 max_unexpected_parts_loading_thread_pool_size;
extern const ServerSettingsUInt64 max_view_num_to_warn;
extern const ServerSettingsUInt64 max_waiting_queries;
extern const ServerSettingsUInt64 memory_worker_period_ms;
extern const ServerSettingsUInt64 merges_mutations_memory_usage_soft_limit;
extern const ServerSettingsDouble merges_mutations_memory_usage_to_ram_ratio;
extern const ServerSettingsString merge_workload;
extern const ServerSettingsUInt64 mmap_cache_size;
extern const ServerSettingsString mutation_workload;
extern const ServerSettingsUInt64 page_cache_chunk_size;
extern const ServerSettingsUInt64 page_cache_mmap_size;
extern const ServerSettingsUInt64 page_cache_size;
extern const ServerSettingsBool page_cache_use_madv_free;
extern const ServerSettingsBool page_cache_use_transparent_huge_pages;
extern const ServerSettingsBool prepare_system_log_tables_on_startup;
extern const ServerSettingsBool show_addresses_in_stack_traces;
extern const ServerSettingsBool shutdown_wait_backups_and_restores;
extern const ServerSettingsUInt64 shutdown_wait_unfinished;
extern const ServerSettingsBool shutdown_wait_unfinished_queries;
extern const ServerSettingsUInt64 storage_connections_soft_limit;
extern const ServerSettingsUInt64 storage_connections_store_limit;
extern const ServerSettingsUInt64 storage_connections_warn_limit;
extern const ServerSettingsUInt64 tables_loader_background_pool_size;
extern const ServerSettingsUInt64 tables_loader_foreground_pool_size;
extern const ServerSettingsString temporary_data_in_cache;
extern const ServerSettingsUInt64 thread_pool_queue_size;
extern const ServerSettingsString tmp_policy;
extern const ServerSettingsUInt64 total_memory_profiler_sample_max_allocation_size;
extern const ServerSettingsUInt64 total_memory_profiler_sample_min_allocation_size;
extern const ServerSettingsUInt64 total_memory_profiler_step;
extern const ServerSettingsDouble total_memory_tracker_sample_probability;
extern const ServerSettingsString uncompressed_cache_policy;
extern const ServerSettingsUInt64 uncompressed_cache_size;
extern const ServerSettingsDouble uncompressed_cache_size_ratio;
extern const ServerSettingsBool use_legacy_mongodb_integration;
}
}
namespace CurrentMetrics
@ -672,7 +789,7 @@ static void initializeAzureSDKLogger(
[[ maybe_unused ]] int server_logs_level)
{
#if USE_AZURE_BLOB_STORAGE
if (!server_settings.enable_azure_sdk_logging)
if (!server_settings[ServerSetting::enable_azure_sdk_logging])
return;
using AzureLogsLevel = Azure::Core::Diagnostics::Logger::Level;
@ -746,9 +863,9 @@ try
ServerSettings server_settings;
server_settings.loadSettingsFromConfig(config());
ASTAlterCommand::setFormatAlterCommandsWithParentheses(server_settings.format_alter_operations_with_parentheses);
ASTAlterCommand::setFormatAlterCommandsWithParentheses(server_settings[ServerSetting::format_alter_operations_with_parentheses]);
StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces);
StackTrace::setShowAddresses(server_settings[ServerSetting::show_addresses_in_stack_traces]);
#if USE_HDFS
/// This will point libhdfs3 to the right location for its config.
@ -794,10 +911,10 @@ try
registerInterpreters();
registerFunctions();
registerAggregateFunctions();
registerTableFunctions(server_settings.use_legacy_mongodb_integration);
registerTableFunctions(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDatabases();
registerStorages(server_settings.use_legacy_mongodb_integration);
registerDictionaries(server_settings.use_legacy_mongodb_integration);
registerStorages(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDictionaries(server_settings[ServerSetting::use_legacy_mongodb_integration]);
registerDisks(/* global_skip_access_check= */ false);
registerFormats();
registerRemoteFileMetadatas();
@ -893,37 +1010,37 @@ try
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
// ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well.
GlobalThreadPool::initialize(
server_settings.max_thread_pool_size,
server_settings.max_thread_pool_free_size,
server_settings.thread_pool_queue_size,
has_trace_collector ? server_settings.global_profiler_real_time_period_ns : 0,
has_trace_collector ? server_settings.global_profiler_cpu_time_period_ns : 0);
server_settings[ServerSetting::max_thread_pool_size],
server_settings[ServerSetting::max_thread_pool_free_size],
server_settings[ServerSetting::thread_pool_queue_size],
has_trace_collector ? server_settings[ServerSetting::global_profiler_real_time_period_ns] : 0,
has_trace_collector ? server_settings[ServerSetting::global_profiler_cpu_time_period_ns] : 0);
if (has_trace_collector)
{
global_context->createTraceCollector();
/// Set up server-wide memory profiler (for total memory tracker).
if (server_settings.total_memory_profiler_step)
total_memory_tracker.setProfilerStep(server_settings.total_memory_profiler_step);
if (server_settings[ServerSetting::total_memory_profiler_step])
total_memory_tracker.setProfilerStep(server_settings[ServerSetting::total_memory_profiler_step]);
if (server_settings.total_memory_tracker_sample_probability > 0.0)
total_memory_tracker.setSampleProbability(server_settings.total_memory_tracker_sample_probability);
if (server_settings[ServerSetting::total_memory_tracker_sample_probability] > 0.0)
total_memory_tracker.setSampleProbability(server_settings[ServerSetting::total_memory_tracker_sample_probability]);
if (server_settings.total_memory_profiler_sample_min_allocation_size)
total_memory_tracker.setSampleMinAllocationSize(server_settings.total_memory_profiler_sample_min_allocation_size);
if (server_settings[ServerSetting::total_memory_profiler_sample_min_allocation_size])
total_memory_tracker.setSampleMinAllocationSize(server_settings[ServerSetting::total_memory_profiler_sample_min_allocation_size]);
if (server_settings.total_memory_profiler_sample_max_allocation_size)
total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
if (server_settings[ServerSetting::total_memory_profiler_sample_max_allocation_size])
total_memory_tracker.setSampleMaxAllocationSize(server_settings[ServerSetting::total_memory_profiler_sample_max_allocation_size]);
}
Poco::ThreadPool server_pool(
/* minCapacity */3,
/* maxCapacity */server_settings.max_connections,
/* maxCapacity */server_settings[ServerSetting::max_connections],
/* idleTime */60,
/* stackSize */POCO_THREAD_STACK_SIZE,
server_settings.global_profiler_real_time_period_ns,
server_settings.global_profiler_cpu_time_period_ns);
server_settings[ServerSetting::global_profiler_real_time_period_ns],
server_settings[ServerSetting::global_profiler_cpu_time_period_ns]);
std::mutex servers_lock;
std::vector<ProtocolServerAdapter> servers;
@ -937,13 +1054,13 @@ try
LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds());
});
MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms);
MemoryWorker memory_worker(global_context->getServerSettings()[ServerSetting::memory_worker_period_ms]);
/// This object will periodically calculate some metrics.
ServerAsynchronousMetrics async_metrics(
global_context,
server_settings.asynchronous_metrics_update_period_s,
server_settings.asynchronous_heavy_metrics_update_period_s,
server_settings[ServerSetting::asynchronous_metrics_update_period_s],
server_settings[ServerSetting::asynchronous_heavy_metrics_update_period_s],
[&]() -> std::vector<ProtocolServerMetrics>
{
std::vector<ProtocolServerMetrics> metrics;
@ -996,7 +1113,7 @@ try
LOG_INFO(log, "Closed all listening sockets.");
if (current_connections > 0)
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings.shutdown_wait_unfinished);
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings[ServerSetting::shutdown_wait_unfinished]);
if (current_connections)
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
@ -1033,47 +1150,47 @@ try
#endif
getIOThreadPool().initialize(
server_settings.max_io_thread_pool_size,
server_settings.max_io_thread_pool_free_size,
server_settings.io_thread_pool_queue_size);
server_settings[ServerSetting::max_io_thread_pool_size],
server_settings[ServerSetting::max_io_thread_pool_free_size],
server_settings[ServerSetting::io_thread_pool_queue_size]);
getBackupsIOThreadPool().initialize(
server_settings.max_backups_io_thread_pool_size,
server_settings.max_backups_io_thread_pool_free_size,
server_settings.backups_io_thread_pool_queue_size);
server_settings[ServerSetting::max_backups_io_thread_pool_size],
server_settings[ServerSetting::max_backups_io_thread_pool_free_size],
server_settings[ServerSetting::backups_io_thread_pool_queue_size]);
getActivePartsLoadingThreadPool().initialize(
server_settings.max_active_parts_loading_thread_pool_size,
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size],
0, // We don't need any threads once all the parts will be loaded
server_settings.max_active_parts_loading_thread_pool_size);
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]);
getOutdatedPartsLoadingThreadPool().initialize(
server_settings.max_outdated_parts_loading_thread_pool_size,
server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size],
0, // We don't need any threads once all the parts will be loaded
server_settings.max_outdated_parts_loading_thread_pool_size);
server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size]);
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(
server_settings.max_active_parts_loading_thread_pool_size
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]
);
getUnexpectedPartsLoadingThreadPool().initialize(
server_settings.max_unexpected_parts_loading_thread_pool_size,
server_settings[ServerSetting::max_unexpected_parts_loading_thread_pool_size],
0, // We don't need any threads once all the parts will be loaded
server_settings.max_unexpected_parts_loading_thread_pool_size);
server_settings[ServerSetting::max_unexpected_parts_loading_thread_pool_size]);
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(
server_settings.max_active_parts_loading_thread_pool_size
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]
);
getPartsCleaningThreadPool().initialize(
server_settings.max_parts_cleaning_thread_pool_size,
server_settings[ServerSetting::max_parts_cleaning_thread_pool_size],
0, // We don't need any threads one all the parts will be deleted
server_settings.max_parts_cleaning_thread_pool_size);
server_settings[ServerSetting::max_parts_cleaning_thread_pool_size]);
auto max_database_replicated_create_table_thread_pool_size = server_settings.max_database_replicated_create_table_thread_pool_size
? server_settings.max_database_replicated_create_table_thread_pool_size
auto max_database_replicated_create_table_thread_pool_size = server_settings[ServerSetting::max_database_replicated_create_table_thread_pool_size]
? server_settings[ServerSetting::max_database_replicated_create_table_thread_pool_size]
: getNumberOfCPUCoresToUse();
getDatabaseReplicatedCreateTablesThreadPool().initialize(
max_database_replicated_create_table_thread_pool_size,
@ -1081,9 +1198,9 @@ try
max_database_replicated_create_table_thread_pool_size);
getDatabaseCatalogDropTablesThreadPool().initialize(
server_settings.database_catalog_drop_table_concurrency,
server_settings[ServerSetting::database_catalog_drop_table_concurrency],
0, // We don't need any threads if there are no DROP queries.
server_settings.database_catalog_drop_table_concurrency);
server_settings[ServerSetting::database_catalog_drop_table_concurrency]);
/// Initialize global local cache for remote filesystem.
if (config().has("local_cache_for_remote_fs"))
@ -1320,18 +1437,18 @@ try
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::serverTimezoneInstance().getTimeZone());
/// Storage with temporary data for processing of heavy queries.
if (!server_settings.tmp_policy.value.empty())
if (!server_settings[ServerSetting::tmp_policy].value.empty())
{
global_context->setTemporaryStoragePolicy(server_settings.tmp_policy, server_settings.max_temporary_data_on_disk_size);
global_context->setTemporaryStoragePolicy(server_settings[ServerSetting::tmp_policy], server_settings[ServerSetting::max_temporary_data_on_disk_size]);
}
else if (!server_settings.temporary_data_in_cache.value.empty())
else if (!server_settings[ServerSetting::temporary_data_in_cache].value.empty())
{
global_context->setTemporaryStorageInCache(server_settings.temporary_data_in_cache, server_settings.max_temporary_data_on_disk_size);
global_context->setTemporaryStorageInCache(server_settings[ServerSetting::temporary_data_in_cache], server_settings[ServerSetting::max_temporary_data_on_disk_size]);
}
else
{
std::string temporary_path = config().getString("tmp_path", path / "tmp/");
global_context->setTemporaryStoragePath(temporary_path, server_settings.max_temporary_data_on_disk_size);
global_context->setTemporaryStoragePath(temporary_path, server_settings[ServerSetting::max_temporary_data_on_disk_size]);
}
/** Directory with 'flags': files indicating temporary settings for the server set by system administrator.
@ -1420,11 +1537,11 @@ try
/// Set up caches.
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * server_settings.cache_size_to_ram_max_ratio);
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * server_settings[ServerSetting::cache_size_to_ram_max_ratio]);
String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
double uncompressed_cache_size_ratio = server_settings.uncompressed_cache_size_ratio;
String uncompressed_cache_policy = server_settings[ServerSetting::uncompressed_cache_policy];
size_t uncompressed_cache_size = server_settings[ServerSetting::uncompressed_cache_size];
double uncompressed_cache_size_ratio = server_settings[ServerSetting::uncompressed_cache_size_ratio];
if (uncompressed_cache_size > max_cache_size)
{
uncompressed_cache_size = max_cache_size;
@ -1432,9 +1549,9 @@ try
}
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
String mark_cache_policy = server_settings.mark_cache_policy;
size_t mark_cache_size = server_settings.mark_cache_size;
double mark_cache_size_ratio = server_settings.mark_cache_size_ratio;
String mark_cache_policy = server_settings[ServerSetting::mark_cache_policy];
size_t mark_cache_size = server_settings[ServerSetting::mark_cache_size];
double mark_cache_size_ratio = server_settings[ServerSetting::mark_cache_size_ratio];
if (mark_cache_size > max_cache_size)
{
mark_cache_size = max_cache_size;
@ -1442,16 +1559,16 @@ try
}
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
size_t page_cache_size = server_settings.page_cache_size;
size_t page_cache_size = server_settings[ServerSetting::page_cache_size];
if (page_cache_size != 0)
global_context->setPageCache(
server_settings.page_cache_chunk_size, server_settings.page_cache_mmap_size,
page_cache_size, server_settings.page_cache_use_madv_free,
server_settings.page_cache_use_transparent_huge_pages);
server_settings[ServerSetting::page_cache_chunk_size], server_settings[ServerSetting::page_cache_mmap_size],
page_cache_size, server_settings[ServerSetting::page_cache_use_madv_free],
server_settings[ServerSetting::page_cache_use_transparent_huge_pages]);
String index_uncompressed_cache_policy = server_settings.index_uncompressed_cache_policy;
size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
double index_uncompressed_cache_size_ratio = server_settings.index_uncompressed_cache_size_ratio;
String index_uncompressed_cache_policy = server_settings[ServerSetting::index_uncompressed_cache_policy];
size_t index_uncompressed_cache_size = server_settings[ServerSetting::index_uncompressed_cache_size];
double index_uncompressed_cache_size_ratio = server_settings[ServerSetting::index_uncompressed_cache_size_ratio];
if (index_uncompressed_cache_size > max_cache_size)
{
index_uncompressed_cache_size = max_cache_size;
@ -1459,9 +1576,9 @@ try
}
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
String index_mark_cache_policy = server_settings.index_mark_cache_policy;
size_t index_mark_cache_size = server_settings.index_mark_cache_size;
double index_mark_cache_size_ratio = server_settings.index_mark_cache_size_ratio;
String index_mark_cache_policy = server_settings[ServerSetting::index_mark_cache_policy];
size_t index_mark_cache_size = server_settings[ServerSetting::index_mark_cache_size];
double index_mark_cache_size_ratio = server_settings[ServerSetting::index_mark_cache_size_ratio];
if (index_mark_cache_size > max_cache_size)
{
index_mark_cache_size = max_cache_size;
@ -1469,7 +1586,7 @@ try
}
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
size_t mmap_cache_size = server_settings.mmap_cache_size;
size_t mmap_cache_size = server_settings[ServerSetting::mmap_cache_size];
if (mmap_cache_size > max_cache_size)
{
mmap_cache_size = max_cache_size;
@ -1489,13 +1606,15 @@ try
global_context->setQueryCache(query_cache_max_size_in_bytes, query_cache_max_entries, query_cache_query_cache_max_entry_size_in_bytes, query_cache_max_entry_size_in_rows);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size;
size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size;
size_t compiled_expression_cache_max_size_in_bytes = server_settings[ServerSetting::compiled_expression_cache_size];
size_t compiled_expression_cache_max_elements = server_settings[ServerSetting::compiled_expression_cache_elements_size];
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
NamedCollectionFactory::instance().loadIfNot();
FileCacheFactory::instance().loadDefaultCaches(config());
/// Initialize main config reloader.
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
@ -1507,7 +1626,7 @@ try
std::optional<CgroupsMemoryUsageObserver> cgroups_memory_usage_observer;
try
{
auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time;
auto wait_time = server_settings[ServerSetting::cgroups_memory_usage_observer_wait_time];
if (wait_time != 0)
cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time));
}
@ -1543,15 +1662,24 @@ try
config().getString("path", DBMS_DEFAULT_PATH),
std::move(main_config_zk_node_cache),
main_config_zk_changed_event,
[&](ConfigurationPtr config, bool initial_loading)
[&, config_file = config().getString("config-file", "config.xml")](ConfigurationPtr config, bool initial_loading)
{
if (!initial_loading)
{
/// Add back "config-file" key which is absent in the reloaded config.
config->setString("config-file", config_file);
/// Apply config updates in global context.
global_context->setConfig(config);
}
Settings::checkNoSettingNamesAtTopLevel(*config, config_path);
ServerSettings new_server_settings;
new_server_settings.loadSettingsFromConfig(*config);
size_t max_server_memory_usage = new_server_settings.max_server_memory_usage;
double max_server_memory_usage_to_ram_ratio = new_server_settings.max_server_memory_usage_to_ram_ratio;
size_t max_server_memory_usage = new_server_settings[ServerSetting::max_server_memory_usage];
double max_server_memory_usage_to_ram_ratio = new_server_settings[ServerSetting::max_server_memory_usage_to_ram_ratio];
size_t current_physical_server_memory = getMemoryAmount(); /// With cgroups, the amount of memory available to the server can be changed dynamically.
size_t default_max_server_memory_usage = static_cast<size_t>(current_physical_server_memory * max_server_memory_usage_to_ram_ratio);
@ -1581,9 +1709,9 @@ try
total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
size_t merges_mutations_memory_usage_soft_limit = new_server_settings.merges_mutations_memory_usage_soft_limit;
size_t merges_mutations_memory_usage_soft_limit = new_server_settings[ServerSetting::merges_mutations_memory_usage_soft_limit];
size_t default_merges_mutations_server_memory_usage = static_cast<size_t>(current_physical_server_memory * new_server_settings.merges_mutations_memory_usage_to_ram_ratio);
size_t default_merges_mutations_server_memory_usage = static_cast<size_t>(current_physical_server_memory * new_server_settings[ServerSetting::merges_mutations_memory_usage_to_ram_ratio]);
if (merges_mutations_memory_usage_soft_limit == 0)
{
merges_mutations_memory_usage_soft_limit = default_merges_mutations_server_memory_usage;
@ -1591,7 +1719,7 @@ try
" ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
formatReadableSizeWithBinarySuffix(current_physical_server_memory),
new_server_settings.merges_mutations_memory_usage_to_ram_ratio);
new_server_settings[ServerSetting::merges_mutations_memory_usage_to_ram_ratio]);
}
else if (merges_mutations_memory_usage_soft_limit > default_merges_mutations_server_memory_usage)
{
@ -1600,7 +1728,7 @@ try
" ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
formatReadableSizeWithBinarySuffix(current_physical_server_memory),
new_server_settings.merges_mutations_memory_usage_to_ram_ratio);
new_server_settings[ServerSetting::merges_mutations_memory_usage_to_ram_ratio]);
}
LOG_INFO(log, "Merges and mutations memory limit is set to {}",
@ -1633,31 +1761,32 @@ try
global_context->setRemoteHostFilter(*config);
global_context->setHTTPHeaderFilter(*config);
global_context->setMaxTableSizeToDrop(new_server_settings.max_table_size_to_drop);
global_context->setMaxPartitionSizeToDrop(new_server_settings.max_partition_size_to_drop);
global_context->setMaxTableNumToWarn(new_server_settings.max_table_num_to_warn);
global_context->setMaxViewNumToWarn(new_server_settings.max_view_num_to_warn);
global_context->setMaxDictionaryNumToWarn(new_server_settings.max_dictionary_num_to_warn);
global_context->setMaxDatabaseNumToWarn(new_server_settings.max_database_num_to_warn);
global_context->setMaxPartNumToWarn(new_server_settings.max_part_num_to_warn);
global_context->setMaxTableSizeToDrop(new_server_settings[ServerSetting::max_table_size_to_drop]);
global_context->setMaxPartitionSizeToDrop(new_server_settings[ServerSetting::max_partition_size_to_drop]);
global_context->setMaxTableNumToWarn(new_server_settings[ServerSetting::max_table_num_to_warn]);
global_context->setMaxViewNumToWarn(new_server_settings[ServerSetting::max_view_num_to_warn]);
global_context->setMaxDictionaryNumToWarn(new_server_settings[ServerSetting::max_dictionary_num_to_warn]);
global_context->setMaxDatabaseNumToWarn(new_server_settings[ServerSetting::max_database_num_to_warn]);
global_context->setMaxPartNumToWarn(new_server_settings[ServerSetting::max_part_num_to_warn]);
/// Only for system.server_settings
global_context->setConfigReloaderInterval(new_server_settings.config_reload_interval_ms);
global_context->setConfigReloaderInterval(new_server_settings[ServerSetting::config_reload_interval_ms]);
SlotCount concurrent_threads_soft_limit = UnlimitedSlots;
if (new_server_settings.concurrent_threads_soft_limit_num > 0 && new_server_settings.concurrent_threads_soft_limit_num < concurrent_threads_soft_limit)
concurrent_threads_soft_limit = new_server_settings.concurrent_threads_soft_limit_num;
if (new_server_settings.concurrent_threads_soft_limit_ratio_to_cores > 0)
if (new_server_settings[ServerSetting::concurrent_threads_soft_limit_num] > 0 && new_server_settings[ServerSetting::concurrent_threads_soft_limit_num] < concurrent_threads_soft_limit)
concurrent_threads_soft_limit = new_server_settings[ServerSetting::concurrent_threads_soft_limit_num];
if (new_server_settings[ServerSetting::concurrent_threads_soft_limit_ratio_to_cores] > 0)
{
auto value = new_server_settings.concurrent_threads_soft_limit_ratio_to_cores * getNumberOfCPUCoresToUse();
auto value = new_server_settings[ServerSetting::concurrent_threads_soft_limit_ratio_to_cores] * getNumberOfCPUCoresToUse();
if (value > 0 && value < concurrent_threads_soft_limit)
concurrent_threads_soft_limit = value;
}
ConcurrencyControl::instance().setMaxConcurrency(concurrent_threads_soft_limit);
LOG_INFO(log, "ConcurrencyControl limit is set to {}", concurrent_threads_soft_limit);
global_context->getProcessList().setMaxSize(new_server_settings.max_concurrent_queries);
global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings.max_concurrent_insert_queries);
global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings.max_concurrent_select_queries);
global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings.max_waiting_queries);
global_context->getProcessList().setMaxSize(new_server_settings[ServerSetting::max_concurrent_queries]);
global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings[ServerSetting::max_concurrent_insert_queries]);
global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings[ServerSetting::max_concurrent_select_queries]);
global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings[ServerSetting::max_waiting_queries]);
if (config->has("keeper_server"))
global_context->updateKeeperConfiguration(*config);
@ -1668,72 +1797,72 @@ try
/// This is done for backward compatibility.
if (global_context->areBackgroundExecutorsInitialized())
{
auto new_pool_size = new_server_settings.background_pool_size;
auto new_ratio = new_server_settings.background_merges_mutations_concurrency_ratio;
auto new_pool_size = new_server_settings[ServerSetting::background_pool_size];
auto new_ratio = new_server_settings[ServerSetting::background_merges_mutations_concurrency_ratio];
global_context->getMergeMutateExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, static_cast<size_t>(new_pool_size * new_ratio));
global_context->getMergeMutateExecutor()->updateSchedulingPolicy(new_server_settings.background_merges_mutations_scheduling_policy.toString());
global_context->getMergeMutateExecutor()->updateSchedulingPolicy(new_server_settings[ServerSetting::background_merges_mutations_scheduling_policy].toString());
}
if (global_context->areBackgroundExecutorsInitialized())
{
auto new_pool_size = new_server_settings.background_move_pool_size;
auto new_pool_size = new_server_settings[ServerSetting::background_move_pool_size];
global_context->getMovesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
}
if (global_context->areBackgroundExecutorsInitialized())
{
auto new_pool_size = new_server_settings.background_fetches_pool_size;
auto new_pool_size = new_server_settings[ServerSetting::background_fetches_pool_size];
global_context->getFetchesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
}
if (global_context->areBackgroundExecutorsInitialized())
{
auto new_pool_size = new_server_settings.background_common_pool_size;
auto new_pool_size = new_server_settings[ServerSetting::background_common_pool_size];
global_context->getCommonExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
}
global_context->getBufferFlushSchedulePool().increaseThreadsCount(new_server_settings.background_buffer_flush_schedule_pool_size);
global_context->getSchedulePool().increaseThreadsCount(new_server_settings.background_schedule_pool_size);
global_context->getMessageBrokerSchedulePool().increaseThreadsCount(new_server_settings.background_message_broker_schedule_pool_size);
global_context->getDistributedSchedulePool().increaseThreadsCount(new_server_settings.background_distributed_schedule_pool_size);
global_context->getBufferFlushSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_buffer_flush_schedule_pool_size]);
global_context->getSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_schedule_pool_size]);
global_context->getMessageBrokerSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_message_broker_schedule_pool_size]);
global_context->getDistributedSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_distributed_schedule_pool_size]);
global_context->getAsyncLoader().setMaxThreads(TablesLoaderForegroundPoolId, new_server_settings.tables_loader_foreground_pool_size);
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundLoadPoolId, new_server_settings.tables_loader_background_pool_size);
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundStartupPoolId, new_server_settings.tables_loader_background_pool_size);
global_context->getAsyncLoader().setMaxThreads(TablesLoaderForegroundPoolId, new_server_settings[ServerSetting::tables_loader_foreground_pool_size]);
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundLoadPoolId, new_server_settings[ServerSetting::tables_loader_background_pool_size]);
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundStartupPoolId, new_server_settings[ServerSetting::tables_loader_background_pool_size]);
getIOThreadPool().reloadConfiguration(
new_server_settings.max_io_thread_pool_size,
new_server_settings.max_io_thread_pool_free_size,
new_server_settings.io_thread_pool_queue_size);
new_server_settings[ServerSetting::max_io_thread_pool_size],
new_server_settings[ServerSetting::max_io_thread_pool_free_size],
new_server_settings[ServerSetting::io_thread_pool_queue_size]);
getBackupsIOThreadPool().reloadConfiguration(
new_server_settings.max_backups_io_thread_pool_size,
new_server_settings.max_backups_io_thread_pool_free_size,
new_server_settings.backups_io_thread_pool_queue_size);
new_server_settings[ServerSetting::max_backups_io_thread_pool_size],
new_server_settings[ServerSetting::max_backups_io_thread_pool_free_size],
new_server_settings[ServerSetting::backups_io_thread_pool_queue_size]);
getActivePartsLoadingThreadPool().reloadConfiguration(
new_server_settings.max_active_parts_loading_thread_pool_size,
new_server_settings[ServerSetting::max_active_parts_loading_thread_pool_size],
0, // We don't need any threads once all the parts will be loaded
new_server_settings.max_active_parts_loading_thread_pool_size);
new_server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]);
getOutdatedPartsLoadingThreadPool().reloadConfiguration(
new_server_settings.max_outdated_parts_loading_thread_pool_size,
new_server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size],
0, // We don't need any threads once all the parts will be loaded
new_server_settings.max_outdated_parts_loading_thread_pool_size);
new_server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size]);
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(
new_server_settings.max_active_parts_loading_thread_pool_size
new_server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]
);
getPartsCleaningThreadPool().reloadConfiguration(
new_server_settings.max_parts_cleaning_thread_pool_size,
new_server_settings[ServerSetting::max_parts_cleaning_thread_pool_size],
0, // We don't need any threads one all the parts will be deleted
new_server_settings.max_parts_cleaning_thread_pool_size);
new_server_settings[ServerSetting::max_parts_cleaning_thread_pool_size]);
global_context->setMergeWorkload(new_server_settings.merge_workload);
global_context->setMutationWorkload(new_server_settings.mutation_workload);
global_context->setMergeWorkload(new_server_settings[ServerSetting::merge_workload]);
global_context->setMutationWorkload(new_server_settings[ServerSetting::mutation_workload]);
if (config->has("resources"))
{
@ -1778,28 +1907,28 @@ try
HTTPConnectionPools::instance().setLimits(
HTTPConnectionPools::Limits{
new_server_settings.disk_connections_soft_limit,
new_server_settings.disk_connections_warn_limit,
new_server_settings.disk_connections_store_limit,
new_server_settings[ServerSetting::disk_connections_soft_limit],
new_server_settings[ServerSetting::disk_connections_warn_limit],
new_server_settings[ServerSetting::disk_connections_store_limit],
},
HTTPConnectionPools::Limits{
new_server_settings.storage_connections_soft_limit,
new_server_settings.storage_connections_warn_limit,
new_server_settings.storage_connections_store_limit,
new_server_settings[ServerSetting::storage_connections_soft_limit],
new_server_settings[ServerSetting::storage_connections_warn_limit],
new_server_settings[ServerSetting::storage_connections_store_limit],
},
HTTPConnectionPools::Limits{
new_server_settings.http_connections_soft_limit,
new_server_settings.http_connections_warn_limit,
new_server_settings.http_connections_store_limit,
new_server_settings[ServerSetting::http_connections_soft_limit],
new_server_settings[ServerSetting::http_connections_warn_limit],
new_server_settings[ServerSetting::http_connections_store_limit],
});
DNSResolver::instance().setFilterSettings(new_server_settings.dns_allow_resolve_names_to_ipv4, new_server_settings.dns_allow_resolve_names_to_ipv6);
DNSResolver::instance().setFilterSettings(new_server_settings[ServerSetting::dns_allow_resolve_names_to_ipv4], new_server_settings[ServerSetting::dns_allow_resolve_names_to_ipv6]);
if (global_context->isServerCompletelyStarted())
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings.cannot_allocate_thread_fault_injection_probability);
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings[ServerSetting::cannot_allocate_thread_fault_injection_probability]);
#if USE_GWP_ASAN
GWPAsan::setForceSampleProbability(new_server_settings.gwp_asan_force_sample_probability);
GWPAsan::setForceSampleProbability(new_server_settings[ServerSetting::gwp_asan_force_sample_probability]);
#endif
ProfileEvents::increment(ProfileEvents::MainConfigLoads);
@ -1995,7 +2124,7 @@ try
});
/// Limit on total number of concurrently executed queries.
global_context->getProcessList().setMaxSize(server_settings.max_concurrent_queries);
global_context->getProcessList().setMaxSize(server_settings[ServerSetting::max_concurrent_queries]);
/// Load global settings from default_profile and system_profile.
global_context->setDefaultProfiles(config());
@ -2004,12 +2133,12 @@ try
/// This is needed to load proper values of background_pool_size etc.
global_context->initializeBackgroundExecutorsIfNeeded();
if (server_settings.async_insert_threads)
if (server_settings[ServerSetting::async_insert_threads])
{
global_context->setAsynchronousInsertQueue(std::make_shared<AsynchronousInsertQueue>(
global_context,
server_settings.async_insert_threads,
server_settings.async_insert_queue_flush_on_shutdown));
server_settings[ServerSetting::async_insert_threads],
server_settings[ServerSetting::async_insert_queue_flush_on_shutdown]));
}
/// Set path for format schema files
@ -2045,7 +2174,7 @@ try
/// context is destroyed.
/// In addition this object has to be created before the loading of the tables.
std::unique_ptr<DNSCacheUpdater> dns_cache_updater;
if (server_settings.disable_internal_dns_cache)
if (server_settings[ServerSetting::disable_internal_dns_cache])
{
/// Disable DNS caching at all
DNSResolver::instance().setDisableCacheFlag();
@ -2053,11 +2182,11 @@ try
}
else
{
DNSResolver::instance().setCacheMaxEntries(server_settings.dns_cache_max_entries);
DNSResolver::instance().setCacheMaxEntries(server_settings[ServerSetting::dns_cache_max_entries]);
/// Initialize a watcher periodically updating DNS cache
dns_cache_updater = std::make_unique<DNSCacheUpdater>(
global_context, server_settings.dns_cache_update_period, server_settings.dns_max_consecutive_failures);
global_context, server_settings[ServerSetting::dns_cache_update_period], server_settings[ServerSetting::dns_max_consecutive_failures]);
}
if (dns_cache_updater)
@ -2065,7 +2194,7 @@ try
/// Set current database name before loading tables and databases because
/// system logs may copy global context.
std::string default_database = server_settings.default_database.toString();
std::string default_database = server_settings[ServerSetting::default_database].toString();
global_context->setCurrentDatabaseNameInGlobalContext(default_database);
LOG_INFO(log, "Loading metadata from {}", path_str);
@ -2101,7 +2230,7 @@ try
waitLoad(TablesLoaderForegroundPoolId, system_startup_tasks);
/// Startup scripts can depend on the system log tables.
if (config().has("startup_scripts") && !server_settings.prepare_system_log_tables_on_startup.changed)
if (config().has("startup_scripts") && !server_settings[ServerSetting::prepare_system_log_tables_on_startup].changed)
global_context->setServerSetting("prepare_system_log_tables_on_startup", true);
/// After attaching system databases we can initialize system log.
@ -2121,7 +2250,7 @@ try
database_catalog.loadMarkedAsDroppedTables();
database_catalog.createBackgroundTasks();
/// Then, load remaining databases (some of them maybe be loaded asynchronously)
load_metadata_tasks = loadMetadata(global_context, default_database, server_settings.async_load_databases);
load_metadata_tasks = loadMetadata(global_context, default_database, server_settings[ServerSetting::async_load_databases]);
/// If we need to convert database engines, disable async tables loading
convertDatabasesEnginesIfNeed(load_metadata_tasks, global_context);
database_catalog.startupBackgroundTasks();
@ -2272,11 +2401,11 @@ try
startup_watch.stop();
ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
CannotAllocateThreadFaultInjector::setFaultProbability(server_settings.cannot_allocate_thread_fault_injection_probability);
CannotAllocateThreadFaultInjector::setFaultProbability(server_settings[ServerSetting::cannot_allocate_thread_fault_injection_probability]);
#if USE_GWP_ASAN
GWPAsan::initFinished();
GWPAsan::setForceSampleProbability(server_settings.gwp_asan_force_sample_probability);
GWPAsan::setForceSampleProbability(server_settings[ServerSetting::gwp_asan_force_sample_probability]);
#endif
try
@ -2326,15 +2455,15 @@ try
/// Wait for unfinished backups and restores.
/// This must be done after closing listening sockets (no more backups/restores) but before ProcessList::killAllQueries
/// (because killAllQueries() will cancel all running backups/restores).
if (server_settings.shutdown_wait_backups_and_restores)
if (server_settings[ServerSetting::shutdown_wait_backups_and_restores])
global_context->waitAllBackupsAndRestores();
/// Killing remaining queries.
if (!server_settings.shutdown_wait_unfinished_queries)
if (!server_settings[ServerSetting::shutdown_wait_unfinished_queries])
global_context->getProcessList().killAllQueries();
if (current_connections)
current_connections = waitServersToFinish(servers, servers_lock, server_settings.shutdown_wait_unfinished);
current_connections = waitServersToFinish(servers, servers_lock, server_settings[ServerSetting::shutdown_wait_unfinished]);
if (current_connections)
LOG_WARNING(log, "Closed connections. But {} remain."
@ -2473,8 +2602,8 @@ void Server::createServers(
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
http_params->setTimeout(settings[Setting::http_receive_timeout]);
http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout);
http_params->setMaxKeepAliveRequests(static_cast<int>(global_context->getServerSettings().max_keep_alive_requests));
http_params->setKeepAliveTimeout(global_context->getServerSettings()[ServerSetting::keep_alive_timeout]);
http_params->setMaxKeepAliveRequests(static_cast<int>(global_context->getServerSettings()[ServerSetting::max_keep_alive_requests]));
Poco::Util::AbstractConfiguration::Keys protocols;
config.keys("protocols", protocols);
@ -2730,7 +2859,7 @@ void Server::createInterserverServers(
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
http_params->setTimeout(settings[Setting::http_receive_timeout]);
http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout);
http_params->setKeepAliveTimeout(global_context->getServerSettings()[ServerSetting::keep_alive_timeout]);
/// Now iterate over interserver_listen_hosts
for (const auto & interserver_listen_host : interserver_listen_hosts)

47
src/Access/AuthenticationData.cpp
View File

@ -117,20 +117,20 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs)
}
void AuthenticationData::setPassword(const String & password_)
void AuthenticationData::setPassword(const String & password_, bool validate)
{
switch (type)
{
case AuthenticationType::PLAINTEXT_PASSWORD:
setPasswordHashBinary(Util::stringToDigest(password_));
setPasswordHashBinary(Util::stringToDigest(password_), validate);
return;
case AuthenticationType::SHA256_PASSWORD:
setPasswordHashBinary(Util::encodeSHA256(password_));
setPasswordHashBinary(Util::encodeSHA256(password_), validate);
return;
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
setPasswordHashBinary(Util::encodeDoubleSHA1(password_));
setPasswordHashBinary(Util::encodeDoubleSHA1(password_), validate);
return;
case AuthenticationType::BCRYPT_PASSWORD:
@ -149,12 +149,12 @@ void AuthenticationData::setPassword(const String & password_)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "setPassword(): authentication type {} not supported", toString(type));
}
void AuthenticationData::setPasswordBcrypt(const String & password_, int workfactor_)
void AuthenticationData::setPasswordBcrypt(const String & password_, int workfactor_, bool validate)
{
if (type != AuthenticationType::BCRYPT_PASSWORD)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot specify bcrypt password for authentication type {}", toString(type));
setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_));
setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_), validate);
}
String AuthenticationData::getPassword() const
@ -165,7 +165,7 @@ String AuthenticationData::getPassword() const
}
void AuthenticationData::setPasswordHashHex(const String & hash)
void AuthenticationData::setPasswordHashHex(const String & hash, bool validate)
{
Digest digest;
digest.resize(hash.size() / 2);
@ -179,7 +179,7 @@ void AuthenticationData::setPasswordHashHex(const String & hash)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read password hash in hex, check for valid characters [0-9a-fA-F] and length");
}
setPasswordHashBinary(digest);
setPasswordHashBinary(digest, validate);
}
@ -195,7 +195,7 @@ String AuthenticationData::getPasswordHashHex() const
}
void AuthenticationData::setPasswordHashBinary(const Digest & hash)
void AuthenticationData::setPasswordHashBinary(const Digest & hash, bool validate)
{
switch (type)
{
@ -217,7 +217,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
{
if (hash.size() != 20)
if (validate && hash.size() != 20)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Password hash for the 'DOUBLE_SHA1_PASSWORD' authentication type has length {} "
"but must be exactly 20 bytes.", hash.size());
@ -231,7 +231,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
/// However the library we use to encode it requires hash string to be 64 characters long,
/// so we also allow the hash of this length.
if (hash.size() != 59 && hash.size() != 60 && hash.size() != 64)
if (validate && hash.size() != 59 && hash.size() != 60 && hash.size() != 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} "
"but must be 59 or 60 bytes.", hash.size());
@ -240,10 +240,13 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
resized.resize(64);
#if USE_BCRYPT
/// Verify that it is a valid hash
int ret = bcrypt_checkpw("", reinterpret_cast<const char *>(resized.data()));
if (ret == -1)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
if (validate)
{
/// Verify that it is a valid hash
int ret = bcrypt_checkpw("", reinterpret_cast<const char *>(resized.data()));
if (ret == -1)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
}
#endif
password_hash = hash;
@ -385,7 +388,7 @@ std::shared_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
}
AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules)
AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & query, ContextPtr context, bool validate)
{
if (query.type && query.type == AuthenticationType::NO_PASSWORD)
return AuthenticationData();
@ -431,7 +434,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
if (!query.type && !context)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get default password type without context");
if (check_password_rules && !context)
if (validate && !context)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot check password complexity rules without context");
if (query.type == AuthenticationType::BCRYPT_PASSWORD && !context)
@ -448,13 +451,13 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
AuthenticationData auth_data(current_type);
if (check_password_rules)
if (validate)
context->getAccessControl().checkPasswordComplexityRules(value);
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
{
int workfactor = context->getAccessControl().getBcryptWorkfactor();
auth_data.setPasswordBcrypt(value, workfactor);
auth_data.setPasswordBcrypt(value, workfactor, validate);
return auth_data;
}
@ -486,7 +489,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
#endif
}
auth_data.setPassword(value);
auth_data.setPassword(value, validate);
return auth_data;
}
@ -498,11 +501,11 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
{
auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value));
auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value), validate);
return auth_data;
}
auth_data.setPasswordHashHex(value);
auth_data.setPasswordHashHex(value, validate);
if (query.type == AuthenticationType::SHA256_PASSWORD && args_size == 2)

10
src/Access/AuthenticationData.h
View File

@ -31,17 +31,17 @@ public:
AuthenticationType getType() const { return type; }
/// Sets the password and encrypt it using the authentication type set in the constructor.
void setPassword(const String & password_);
void setPassword(const String & password_, bool validate);
/// Returns the password. Allowed to use only for Type::PLAINTEXT_PASSWORD.
String getPassword() const;
/// Sets the password as a string of hexadecimal digits.
void setPasswordHashHex(const String & hash);
void setPasswordHashHex(const String & hash, bool validate);
String getPasswordHashHex() const;
/// Sets the password in binary form.
void setPasswordHashBinary(const Digest & hash);
void setPasswordHashBinary(const Digest & hash, bool validate);
const Digest & getPasswordHashBinary() const { return password_hash; }
/// Sets the salt in String form.
@ -49,7 +49,7 @@ public:
String getSalt() const;
/// Sets the password using bcrypt hash with specified workfactor
void setPasswordBcrypt(const String & password_, int workfactor_);
void setPasswordBcrypt(const String & password_, int workfactor_, bool validate);
/// Sets the server name for authentication type LDAP.
const String & getLDAPServerName() const { return ldap_server_name; }
@ -77,7 +77,7 @@ public:
friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs);
friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); }
static AuthenticationData fromAST(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules);
static AuthenticationData fromAST(const ASTAuthenticationData & query, ContextPtr context, bool validate);
std::shared_ptr<ASTAuthenticationData> toAST() const;
struct Util

7
src/Access/UsersConfigAccessStorage.cpp
View File

@ -121,6 +121,7 @@ namespace
bool allow_no_password,
bool allow_plaintext_password)
{
const bool validate = true;
auto user = std::make_shared<User>();
user->setName(user_name);
String user_config = "users." + user_name;
@ -157,17 +158,17 @@ namespace
if (has_password_plaintext)
{
user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD);
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"));
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"), validate);
}
else if (has_password_sha256_hex)
{
user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD);
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"), validate);
}
else if (has_password_double_sha1_hex)
{
user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD);
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"), validate);
}
else if (has_ldap)
{

10
src/AggregateFunctions/AggregateFunctionGroupArray.cpp
View File

@ -33,6 +33,12 @@ namespace DB
{
struct Settings;
namespace ServerSetting
{
extern const ServerSettingsGroupArrayActionWhenLimitReached aggregate_function_group_array_action_when_limit_is_reached;
extern const ServerSettingsUInt64 aggregate_function_group_array_max_element_size;
}
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
@ -746,7 +752,7 @@ inline AggregateFunctionPtr createAggregateFunctionGroupArrayImpl(const DataType
size_t getMaxArraySize()
{
if (auto context = Context::getGlobalContextInstance())
return context->getServerSettings().aggregate_function_group_array_max_element_size;
return context->getServerSettings()[ServerSetting::aggregate_function_group_array_max_element_size];
return 0xFFFFFF;
}
@ -754,7 +760,7 @@ size_t getMaxArraySize()
bool discardOnLimitReached()
{
if (auto context = Context::getGlobalContextInstance())
return context->getServerSettings().aggregate_function_group_array_action_when_limit_is_reached
return context->getServerSettings()[ServerSetting::aggregate_function_group_array_action_when_limit_is_reached]
== GroupArrayActionWhenLimitReached::DISCARD;
return false;

352
src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp
View File

@ -28,11 +28,36 @@ namespace ErrorCodes
namespace
{
constexpr size_t max_events = 32;
constexpr size_t MAX_EVENTS = 32;
template <typename T>
void mergeEventsList(T & events_list, size_t prefix_size, bool prefix_sorted, bool suffix_sorted)
{
/// either sort whole container or do so partially merging ranges afterwards
if (!prefix_sorted && !suffix_sorted)
std::stable_sort(std::begin(events_list), std::end(events_list));
else
{
const auto begin = std::begin(events_list);
const auto middle = std::next(begin, prefix_size);
const auto end = std::end(events_list);
if (!prefix_sorted)
std::stable_sort(begin, middle);
if (!suffix_sorted)
std::stable_sort(middle, end);
std::inplace_merge(begin, middle, end);
}
}
template <typename T>
struct AggregateFunctionWindowFunnelData
{
static constexpr bool strict_once_enabled = false;
using TimestampEvent = std::pair<T, UInt8>;
using TimestampEvents = PODArrayWithStackMemory<TimestampEvent, 64>;
@ -66,24 +91,7 @@ struct AggregateFunctionWindowFunnelData
events_list.insert(std::begin(other.events_list), std::end(other.events_list));
/// either sort whole container or do so partially merging ranges afterwards
if (!sorted && !other.sorted)
std::stable_sort(std::begin(events_list), std::end(events_list));
else
{
const auto begin = std::begin(events_list);
const auto middle = std::next(begin, size);
const auto end = std::end(events_list);
if (!sorted)
std::stable_sort(begin, middle);
if (!other.sorted)
std::stable_sort(middle, end);
std::inplace_merge(begin, middle, end);
}
mergeEventsList(events_list, size, sorted, other.sorted);
sorted = true;
}
@ -133,6 +141,131 @@ struct AggregateFunctionWindowFunnelData
}
};
template <typename T>
struct AggregateFunctionWindowFunnelStrictOnceData
{
static constexpr bool strict_once_enabled = true;
struct TimestampEvent
{
T timestamp;
UInt8 event_type;
UInt64 unique_id;
TimestampEvent(T timestamp_, UInt8 event_type_, UInt64 unique_id_)
: timestamp(timestamp_), event_type(event_type_), unique_id(unique_id_) {}
bool operator<(const TimestampEvent & other) const
{
return std::tie(timestamp, event_type, unique_id) < std::tie(other.timestamp, other.event_type, other.unique_id);
}
bool operator<=(const TimestampEvent & other) const
{
return std::tie(timestamp, event_type, unique_id) <= std::tie(other.timestamp, other.event_type, other.unique_id);
}
};
using TimestampEvents = PODArrayWithStackMemory<TimestampEvent, 64>;
TimestampEvents events_list;
/// Next unique identifier for events
/// Used to distinguish events with the same timestamp that matches several conditions.
UInt64 next_unique_id = 1;
bool sorted = true;
size_t size() const
{
return events_list.size();
}
void advanceId()
{
++next_unique_id;
}
void add(T timestamp, UInt8 event_type)
{
TimestampEvent new_event(timestamp, event_type, next_unique_id);
/// Check if the new event maintains the sorted order
if (sorted && !events_list.empty())
sorted = events_list.back() <= new_event;
events_list.push_back(new_event);
}
void merge(const AggregateFunctionWindowFunnelStrictOnceData & other)
{
if (other.events_list.empty())
return;
const auto current_size = events_list.size();
UInt64 new_next_unique_id = next_unique_id;
events_list.reserve(current_size + other.events_list.size());
for (auto other_event : other.events_list)
{
/// Assign unique IDs to the new events to prevent conflicts
other_event.unique_id += next_unique_id;
new_next_unique_id = std::max(new_next_unique_id, other_event.unique_id + 1);
events_list.push_back(other_event);
}
next_unique_id = new_next_unique_id;
mergeEventsList(events_list, current_size, sorted, other.sorted);
sorted = true;
}
void sort()
{
if (!sorted)
{
std::stable_sort(std::begin(events_list), std::end(events_list));
sorted = true;
}
}
void serialize(WriteBuffer & buf) const
{
writeBinary(sorted, buf);
writeBinary(events_list.size(), buf);
for (const auto & event : events_list)
{
writeBinary(event.timestamp, buf);
writeBinary(event.event_type, buf);
writeBinary(event.unique_id, buf);
}
}
void deserialize(ReadBuffer & buf)
{
readBinary(sorted, buf);
size_t events_size;
readBinary(events_size, buf);
if (events_size > 100'000'000) /// Arbitrary limit to prevent excessive memory allocation
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large size of the state of windowFunnel");
events_list.clear();
events_list.reserve(events_size);
T timestamp;
UInt8 event_type;
UInt64 unique_id = 0;
for (size_t i = 0; i < events_size; ++i)
{
readBinary(timestamp, buf);
readBinary(event_type, buf);
readBinary(unique_id, buf);
next_unique_id = std::max(next_unique_id, unique_id + 1);
events_list.emplace_back(timestamp, event_type, unique_id);
}
}
};
/** Calculates the max event level in a sliding window.
* The max size of events is 32, that's enough for funnel analytics
*
@ -160,22 +293,15 @@ private:
/// The level path must be 1---2---3---...---check_events_size, find the max event level that satisfied the path in the sliding window.
/// If found, returns the max event level, else return 0.
/// The algorithm works in O(n) time, but the overall function works in O(n * log(n)) due to sorting.
UInt8 getEventLevel(Data & data) const
UInt8 getEventLevelNonStrictOnce(const AggregateFunctionWindowFunnelData<T>::TimestampEvents & events_list) const
{
if (data.size() == 0)
return 0;
if (!strict_order && events_size == 1)
return 1;
data.sort();
/// events_timestamp stores the timestamp of the first and previous i-th level event happen within time window
std::vector<std::optional<std::pair<UInt64, UInt64>>> events_timestamp(events_size);
bool first_event = false;
for (size_t i = 0; i < data.events_list.size(); ++i)
for (size_t i = 0; i < events_list.size(); ++i)
{
const T & timestamp = data.events_list[i].first;
const auto & event_idx = data.events_list[i].second - 1;
const T & timestamp = events_list[i].first;
const auto & event_idx = events_list[i].second - 1;
if (strict_order && event_idx == -1)
{
if (first_event)
@ -189,7 +315,7 @@ private:
}
else if (strict_deduplication && events_timestamp[event_idx].has_value())
{
return data.events_list[i - 1].second;
return events_list[i - 1].second;
}
else if (strict_order && first_event && !events_timestamp[event_idx - 1].has_value())
{
@ -222,6 +348,126 @@ private:
return 0;
}
UInt8 getEventLevelStrictOnce(const AggregateFunctionWindowFunnelStrictOnceData<T>::TimestampEvents & events_list) const
{
/// Stores the timestamp of the first and last i-th level event happen within time window
struct EventMatchTimeWindow
{
UInt64 first_timestamp;
UInt64 last_timestamp;
std::array<UInt64, MAX_EVENTS> event_path;
EventMatchTimeWindow() = default;
EventMatchTimeWindow(UInt64 first_ts, UInt64 last_ts)
: first_timestamp(first_ts), last_timestamp(last_ts) {}
};
/// We track all possible event sequences up to the current event.
/// It's required because one event can meet several conditions.
/// For example: for events 'start', 'a', 'b', 'a', 'end'.
/// The second occurrence of 'a' should be counted only once in one sequence.
/// However, we do not know in advance if the next event will be 'b' or 'end', so we try to keep both paths.
std::vector<std::list<EventMatchTimeWindow>> event_sequences(events_size);
bool has_first_event = false;
for (size_t i = 0; i < events_list.size(); ++i)
{
const auto & current_event = events_list[i];
auto timestamp = current_event.timestamp;
Int64 event_idx = current_event.event_type - 1;
UInt64 unique_id = current_event.unique_id;
if (strict_order && event_idx == -1)
{
if (has_first_event)
break;
else
continue;
}
else if (event_idx == 0)
{
auto & event_seq = event_sequences[0].emplace_back(timestamp, timestamp);
event_seq.event_path[0] = unique_id;
has_first_event = true;
}
else if (strict_deduplication && !event_sequences[event_idx].empty())
{
return events_list[i - 1].event_type;
}
else if (strict_order && has_first_event && event_sequences[event_idx - 1].empty())
{
for (size_t event = 0; event < event_sequences.size(); ++event)
{
if (event_sequences[event].empty())
return event;
}
}
else if (!event_sequences[event_idx - 1].empty())
{
auto & prev_level = event_sequences[event_idx - 1];
for (auto it = prev_level.begin(); it != prev_level.end();)
{
auto first_ts = it->first_timestamp;
bool time_matched = timestamp <= first_ts + window;
if (!time_matched && prev_level.size() > 1)
{
// Remove old events that are out of the window, but keep at least one
it = prev_level.erase(it);
continue;
}
auto prev_path = it->event_path;
chassert(event_idx > 0);
/// Ensure the unique_id hasn't been used in the path already
for (size_t j = 0; j < static_cast<size_t>(event_idx); ++j)
{
if (!time_matched)
break;
time_matched = prev_path[j] != unique_id;
}
if (time_matched && strict_increase)
time_matched = it->last_timestamp < timestamp;
if (time_matched)
{
prev_path[event_idx] = unique_id;
auto & new_seq = event_sequences[event_idx].emplace_back(first_ts, timestamp);
new_seq.event_path = std::move(prev_path);
if (event_idx + 1 == events_size)
return events_size;
}
++it;
}
}
}
for (size_t event = event_sequences.size(); event > 0; --event)
{
if (!event_sequences[event - 1].empty())
return event;
}
return 0;
}
UInt8 getEventLevel(Data & data) const
{
if (data.size() == 0)
return 0;
if (!strict_order && events_size == 1)
return 1;
data.sort();
if constexpr (Data::strict_once_enabled)
return getEventLevelStrictOnce(data.events_list);
else
return getEventLevelNonStrictOnce(data.events_list);
}
public:
String getName() const override
{
@ -246,6 +492,9 @@ public:
strict_order = true;
else if (option == "strict_increase")
strict_increase = true;
else if (option == "strict_once")
/// Checked in factory
chassert(Data::strict_once_enabled);
else if (option == "strict")
throw Exception(ErrorCodes::BAD_ARGUMENTS, "strict is replaced with strict_deduplication in Aggregate function {}", getName());
else
@ -272,6 +521,9 @@ public:
if (strict_order && !has_event)
this->data(place).add(timestamp, 0);
if constexpr (Data::strict_once_enabled)
this->data(place).advanceId();
}
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
@ -296,7 +548,6 @@ public:
};
template <template <typename> class Data>
AggregateFunctionPtr
createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes & arguments, const Array & params, const Settings *)
{
@ -309,7 +560,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Aggregate function {} requires one timestamp argument and at least one event condition.", name);
if (arguments.size() > max_events + 1)
if (arguments.size() > MAX_EVENTS + 1)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many event arguments for aggregate function {}", name);
for (const auto i : collections::range(1, arguments.size()))
@ -321,16 +572,29 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
cond_arg->getName(), toString(i + 1), name);
}
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, Data>(*arguments[0], arguments, params));
WhichDataType which(arguments.front().get());
if (res)
return res;
if (which.isDate())
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDate::FieldType, Data<DataTypeDate::FieldType>>>(arguments, params);
if (which.isDateTime())
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDateTime::FieldType, Data<DataTypeDateTime::FieldType>>>(
arguments, params);
bool strict_once = params.size() > 1 && std::any_of(params.begin() + 1, params.end(), [](const auto & f) { return f.template safeGet<String>() == "strict_once"; });
if (strict_once)
{
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, AggregateFunctionWindowFunnelStrictOnceData>(*arguments[0], arguments, params));
WhichDataType which(arguments.front().get());
if (res)
return res;
if (which.isDate())
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDate::FieldType, AggregateFunctionWindowFunnelStrictOnceData<DataTypeDate::FieldType>>>(arguments, params);
if (which.isDateTime())
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDateTime::FieldType, AggregateFunctionWindowFunnelStrictOnceData<DataTypeDateTime::FieldType>>>(arguments, params);
}
else
{
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, AggregateFunctionWindowFunnelData>(*arguments[0], arguments, params));
WhichDataType which(arguments.front().get());
if (res)
return res;
if (which.isDate())
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDate::FieldType, AggregateFunctionWindowFunnelData<DataTypeDate::FieldType>>>(arguments, params);
if (which.isDateTime())
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDateTime::FieldType, AggregateFunctionWindowFunnelData<DataTypeDateTime::FieldType>>>(arguments, params);
}
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of first argument of aggregate function {}, must "
"be Unsigned Number, Date, DateTime", arguments.front().get()->getName(), name);
@ -340,7 +604,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory & factory)
{
factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel<AggregateFunctionWindowFunnelData>);
factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel);
}
}

9
src/AggregateFunctions/IAggregateFunction.cpp
View File

@ -10,6 +10,15 @@ DataTypePtr IAggregateFunction::getStateType() const
return std::make_shared<DataTypeAggregateFunction>(shared_from_this(), argument_types, parameters);
}
DataTypePtr IAggregateFunction::getNormalizedStateType() const
{
DataTypes normalized_argument_types;
normalized_argument_types.reserve(argument_types.size());
for (const auto & arg : argument_types)
normalized_argument_types.emplace_back(arg->getNormalizedType());
return std::make_shared<DataTypeAggregateFunction>(shared_from_this(), normalized_argument_types, parameters);
}
String IAggregateFunction::getDescription() const
{
String description;

2
src/AggregateFunctions/IAggregateFunction.h
View File

@ -73,7 +73,7 @@ public:
virtual DataTypePtr getStateType() const;
/// Same as the above but normalize state types so that variants with the same binary representation will use the same type.
virtual DataTypePtr getNormalizedStateType() const { return getStateType(); }
virtual DataTypePtr getNormalizedStateType() const;
/// Returns true if two aggregate functions have the same state representation in memory and the same serialization,
/// so state of one aggregate function can be safely used with another.

23
src/AggregateFunctions/WindowFunction.h
View File

@ -78,11 +78,6 @@ struct WindowFunction : public IAggregateFunctionHelper<WindowFunction>, public
}
String getName() const override { return name; }
void create(AggregateDataPtr __restrict) const override { }
void destroy(AggregateDataPtr __restrict) const noexcept override { }
bool hasTrivialDestructor() const override { return true; }
size_t sizeOfData() const override { return 0; }
size_t alignOfData() const override { return 1; }
void add(AggregateDataPtr __restrict, const IColumn **, size_t, Arena *) const override { fail(); }
void merge(AggregateDataPtr __restrict, ConstAggregateDataPtr, Arena *) const override { fail(); }
void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional<size_t>) const override { fail(); }
@ -90,6 +85,22 @@ struct WindowFunction : public IAggregateFunctionHelper<WindowFunction>, public
void insertResultInto(AggregateDataPtr __restrict, IColumn &, Arena *) const override { fail(); }
};
struct StatelessWindowFunction : public WindowFunction
{
StatelessWindowFunction(
const std::string & name_, const DataTypes & argument_types_, const Array & parameters_, const DataTypePtr & result_type_)
: WindowFunction(name_, argument_types_, parameters_, result_type_)
{
}
size_t sizeOfData() const override { return 0; }
size_t alignOfData() const override { return 1; }
void create(AggregateDataPtr __restrict) const override { }
void destroy(AggregateDataPtr __restrict) const noexcept override { }
bool hasTrivialDestructor() const override { return true; }
};
template <typename State>
struct StatefulWindowFunction : public WindowFunction
{
@ -100,7 +111,7 @@ struct StatefulWindowFunction : public WindowFunction
}
size_t sizeOfData() const override { return sizeof(State); }
size_t alignOfData() const override { return 1; }
size_t alignOfData() const override { return alignof(State); }
void create(AggregateDataPtr __restrict place) const override { new (place) State(); }

6
src/Analyzer/Passes/CrossToInnerJoinPass.cpp
View File

@ -34,7 +34,7 @@ namespace ErrorCodes
namespace
{
void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other)
void extractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other)
{
auto * func = node->as<FunctionNode>();
if (!func)
@ -52,7 +52,7 @@ void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi
else if (func->getFunctionName() == "and")
{
for (const auto & arg : args)
exctractJoinConditions(arg, equi_conditions, other);
extractJoinConditions(arg, equi_conditions, other);
}
else
{
@ -118,7 +118,7 @@ public:
QueryTreeNodes equi_conditions;
QueryTreeNodes other_conditions;
exctractJoinConditions(where_condition, equi_conditions, other_conditions);
extractJoinConditions(where_condition, equi_conditions, other_conditions);
bool can_convert_cross_to_inner = false;
for (auto & condition : equi_conditions)
{

20
src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
View File

@ -184,6 +184,26 @@ public:
, join_node(join_node_)
{}
bool needChildVisit(const QueryTreeNodePtr & parent, const QueryTreeNodePtr &)
{
/** Optimization can change the value of some expression from NULL to FALSE.
* For example:
* when `a` is `NULL`, the expression `a = b AND a IS NOT NULL` returns `NULL`
* and it will be optimized to `a = b`, which returns `FALSE`.
* This is valid for JOIN ON condition and for the functions `AND`/`OR` inside it.
* (When we replace `AND`/`OR` operands from `NULL` to `FALSE`, the result value can also change only from `NULL` to `FALSE`)
* However, in the general case, the result can be wrong.
* For example, for NOT: `NOT NULL` is `NULL`, but `NOT FALSE` is `TRUE`.
* Therefore, optimize only top-level expression or expressions inside `AND`/`OR`.
*/
if (const auto * function_node = parent->as<FunctionNode>())
{
const auto & func_name = function_node->getFunctionName();
return func_name == "or" || func_name == "and";
}
return parent->getNodeType() == QueryTreeNodeType::LIST;
}
void enterImpl(QueryTreeNodePtr & node)
{
auto * function_node = node->as<FunctionNode>();

8
src/Analyzer/Resolve/IdentifierResolver.cpp
View File

@ -432,6 +432,14 @@ QueryTreeNodePtr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalo
else
storage = DatabaseCatalog::instance().tryGetTable(storage_id, context);
if (!storage && storage_id.hasUUID())
{
// If `storage_id` has UUID, it is possible that the UUID is removed from `DatabaseCatalog` after `context->resolveStorageID(storage_id)`
// We try to get the table with the database name and the table name.
auto database = DatabaseCatalog::instance().tryGetDatabase(storage_id.getDatabaseName());
if (database)
storage = database->tryGetTable(table_name, context);
}
if (!storage)
return {};

2
src/Analyzer/Resolve/QueryAnalyzer.cpp
View File

@ -103,6 +103,7 @@ namespace Setting
extern const SettingsBool single_join_prefer_left_table;
extern const SettingsBool transform_null_in;
extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions;
extern const SettingsBool use_concurrency_control;
}
@ -588,6 +589,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
PullingAsyncPipelineExecutor executor(io.pipeline);
io.pipeline.setProgressCallback(context->getProgressCallback());
io.pipeline.setProcessListElement(context->getProcessListElement());
io.pipeline.setConcurrencyControl(context->getSettingsRef()[Setting::use_concurrency_control]);
Block block;

8
src/Backups/BackupIO_Disk.cpp
View File

@ -20,7 +20,7 @@ BackupReaderDisk::~BackupReaderDisk() = default;
bool BackupReaderDisk::fileExists(const String & file_name)
{
return disk->exists(root_path / file_name);
return disk->existsFile(root_path / file_name);
}
UInt64 BackupReaderDisk::getFileSize(const String & file_name)
@ -68,7 +68,7 @@ BackupWriterDisk::~BackupWriterDisk() = default;
bool BackupWriterDisk::fileExists(const String & file_name)
{
return disk->exists(root_path / file_name);
return disk->existsFile(root_path / file_name);
}
UInt64 BackupWriterDisk::getFileSize(const String & file_name)
@ -91,7 +91,7 @@ std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_nam
void BackupWriterDisk::removeFile(const String & file_name)
{
disk->removeFileIfExists(root_path / file_name);
if (disk->isDirectory(root_path) && disk->isDirectoryEmpty(root_path))
if (disk->existsDirectory(root_path) && disk->isDirectoryEmpty(root_path))
disk->removeDirectory(root_path);
}
@ -99,7 +99,7 @@ void BackupWriterDisk::removeFiles(const Strings & file_names)
{
for (const auto & file_name : file_names)
disk->removeFileIfExists(root_path / file_name);
if (disk->isDirectory(root_path) && disk->isDirectoryEmpty(root_path))
if (disk->existsDirectory(root_path) && disk->isDirectoryEmpty(root_path))
disk->removeDirectory(root_path);
}

96
src/Backups/BackupIO_S3.cpp
View File

@ -10,6 +10,7 @@
#include <IO/WriteBufferFromS3.h>
#include <IO/HTTPHeaderEntries.h>
#include <IO/S3/copyS3File.h>
#include <IO/S3/deleteFileFromS3.h>
#include <IO/S3/Client.h>
#include <IO/S3/Credentials.h>
#include <Disks/IDisk.h>
@ -117,12 +118,6 @@ namespace
throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
return outcome.GetResult().GetContents();
}
bool isNotFoundError(Aws::S3::S3Errors error)
{
return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
|| error == Aws::S3::S3Errors::NO_SUCH_KEY;
}
}
@ -236,6 +231,7 @@ BackupWriterS3::BackupWriterS3(
: BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterS3"))
, s3_uri(s3_uri_)
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::S3, MetadataStorageType::None, s3_uri.endpoint, false, false}
, s3_capabilities(getCapabilitiesFromConfig(context_->getConfigRef(), "s3"))
{
s3_settings.loadFromConfig(context_->getConfigRef(), "s3", context_->getSettingsRef());
@ -358,92 +354,22 @@ std::unique_ptr<WriteBuffer> BackupWriterS3::writeFile(const String & file_name)
void BackupWriterS3::removeFile(const String & file_name)
{
S3::DeleteObjectRequest request;
request.SetBucket(s3_uri.bucket);
auto key = fs::path(s3_uri.key) / file_name;
request.SetKey(key);
auto outcome = client->DeleteObject(request);
if (blob_storage_log)
{
blob_storage_log->addEvent(
BlobStorageLogElement::EventType::Delete,
s3_uri.bucket, key, /* local_path */ "", /* data_size */ 0,
outcome.IsSuccess() ? nullptr : &outcome.GetError());
}
if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType()))
throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
deleteFileFromS3(client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, /* if_exists = */ false,
blob_storage_log);
}
void BackupWriterS3::removeFiles(const Strings & file_names)
{
try
{
if (!supports_batch_delete.has_value() || supports_batch_delete.value() == true)
{
removeFilesBatch(file_names);
supports_batch_delete = true;
}
else
{
for (const auto & file_name : file_names)
removeFile(file_name);
}
}
catch (const Exception &)
{
if (!supports_batch_delete.has_value())
{
supports_batch_delete = false;
LOG_TRACE(log, "DeleteObjects is not supported. Retrying with plain DeleteObject.");
Strings keys;
keys.reserve(file_names.size());
for (const String & file_name : file_names)
keys.push_back(fs::path(s3_uri.key) / file_name);
for (const auto & file_name : file_names)
removeFile(file_name);
}
else
throw;
}
}
void BackupWriterS3::removeFilesBatch(const Strings & file_names)
{
/// One call of DeleteObjects() cannot remove more than 1000 keys.
size_t chunk_size_limit = 1000;
size_t batch_size = 1000;
size_t current_position = 0;
while (current_position < file_names.size())
{
std::vector<Aws::S3::Model::ObjectIdentifier> current_chunk;
for (; current_position < file_names.size() && current_chunk.size() < chunk_size_limit; ++current_position)
{
Aws::S3::Model::ObjectIdentifier obj;
obj.SetKey(fs::path(s3_uri.key) / file_names[current_position]);
current_chunk.push_back(obj);
}
Aws::S3::Model::Delete delkeys;
delkeys.SetObjects(current_chunk);
S3::DeleteObjectsRequest request;
request.SetBucket(s3_uri.bucket);
request.SetDelete(delkeys);
auto outcome = client->DeleteObjects(request);
if (blob_storage_log)
{
const auto * outcome_error = outcome.IsSuccess() ? nullptr : &outcome.GetError();
auto time_now = std::chrono::system_clock::now();
for (const auto & obj : current_chunk)
blob_storage_log->addEvent(BlobStorageLogElement::EventType::Delete, s3_uri.bucket, obj.GetKey(),
/* local_path */ "", /* data_size */ 0, outcome_error, time_now);
}
if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType()))
throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
}
deleteFilesFromS3(client, s3_uri.bucket, keys, /* if_exists = */ false,
s3_capabilities, batch_size, blob_storage_log);
}
}

5
src/Backups/BackupIO_S3.h
View File

@ -10,6 +10,7 @@
#include <IO/S3Settings.h>
#include <Interpreters/Context_fwd.h>
#include <IO/S3/BlobStorageLogWriter.h>
#include <IO/S3/S3Capabilities.h>
namespace DB
{
@ -76,14 +77,12 @@ public:
private:
std::unique_ptr<ReadBuffer> readFile(const String & file_name, size_t expected_file_size) override;
void removeFilesBatch(const Strings & file_names);
const S3::URI s3_uri;
const DataSourceDescription data_source_description;
S3Settings s3_settings;
std::shared_ptr<S3::Client> client;
std::optional<bool> supports_batch_delete;
S3Capabilities s3_capabilities;
BlobStorageLogWriterPtr blob_storage_log;
};

10
src/Backups/DDLAdjustingForBackupVisitor.cpp
View File

@ -11,6 +11,12 @@
namespace DB
{
namespace ServerSetting
{
extern const ServerSettingsString default_replica_name;
extern const ServerSettingsString default_replica_path;
}
namespace
{
void visitStorageSystemTableEngine(ASTStorage &, const DDLAdjustingForBackupVisitor::Data & data)
@ -55,8 +61,8 @@ namespace
zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}");
}
const auto & server_settings = data.global_context->getServerSettings();
if ((zookeeper_path_arg == server_settings.default_replica_path.value)
&& (replica_name_arg == server_settings.default_replica_name.value)
if ((zookeeper_path_arg == server_settings[ServerSetting::default_replica_path].value)
&& (replica_name_arg == server_settings[ServerSetting::default_replica_name].value)
&& ((engine_args.size() == 2) || !engine_args[2]->as<ASTLiteral>()))
{
engine_args.erase(engine_args.begin(), engine_args.begin() + 2);

6
src/BridgeHelper/LibraryBridgeHelper.cpp
View File

@ -10,6 +10,10 @@ namespace Setting
{
extern const SettingsSeconds http_receive_timeout;
}
namespace ServerSetting
{
extern const ServerSettingsSeconds keep_alive_timeout;
}
LibraryBridgeHelper::LibraryBridgeHelper(ContextPtr context_)
: IBridgeHelper(context_)
@ -18,7 +22,7 @@ LibraryBridgeHelper::LibraryBridgeHelper(ContextPtr context_)
, http_timeout(context_->getGlobalContext()->getSettingsRef()[Setting::http_receive_timeout].value)
, bridge_host(config.getString("library_bridge.host", DEFAULT_HOST))
, bridge_port(config.getUInt("library_bridge.port", DEFAULT_PORT))
, http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_->getSettingsRef(), context_->getServerSettings().keep_alive_timeout))
, http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_->getSettingsRef(), context_->getServerSettings()))
{
}

Some files were not shown because too many files have changed in this diff Show More