diff --git a/.gitmodules b/.gitmodules index c46b1c736fc..979f347e6ad 100644 --- a/.gitmodules +++ b/.gitmodules @@ -265,6 +265,9 @@ [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git +[submodule "contrib/nats-io"] + path = contrib/nats-io + url = https://github.com/ClickHouse/nats.c.git [submodule "contrib/vectorscan"] path = contrib/vectorscan url = https://github.com/VectorCamp/vectorscan.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c7b732c68c..4180ae9cce7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,11 +223,25 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") endif () endif() +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" + OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" + OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") + set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT ON) +else() + set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT OFF) +endif() +# Provides faster linking and lower binary size. +# Tradeoff is the inability to debug some source files with e.g. gdb +# (empty stack frames and no local variables)." +option(OMIT_HEAVY_DEBUG_SYMBOLS + "Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)" + ${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT}) + if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") set(USE_DEBUG_HELPERS ON) endif() - option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS}) + option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF) if (NOT BUILD_STANDALONE_KEEPER) option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" ON) diff --git a/PreLoad.cmake b/PreLoad.cmake index 8ef93a7aa51..95f65b85f7f 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -62,9 +62,10 @@ execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH) # By default, prefer clang on Linux # But note, that you still may change the compiler with -DCMAKE_C_COMPILER/-DCMAKE_CXX_COMPILER. if (OS MATCHES "Linux" - # some build systems may use CC/CXX env variables AND "$ENV{CC}" STREQUAL "" - AND "$ENV{CXX}" STREQUAL "") + AND "$ENV{CXX}" STREQUAL "" + AND NOT DEFINED CMAKE_C_COMPILER + AND NOT DEFINED CMAKE_CXX_COMPILER) find_program(CLANG_PATH clang) if (CLANG_PATH) set(CMAKE_C_COMPILER "clang" CACHE INTERNAL "") @@ -87,8 +88,7 @@ if (OS MATCHES "Linux" set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)") set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "") -else () + else () message (FATAL_ERROR "Unsupported architecture: ${ARCH}") endif () - endif() diff --git a/SECURITY.md b/SECURITY.md index 467bdb51466..81d2fd18fb2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -37,7 +37,7 @@ The following versions of ClickHouse server are currently being supported with s We're extremely grateful for security researchers and users that report vulnerabilities to the ClickHouse Open Source Community. All reports are thoroughly investigated by developers. -To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com). +To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com). We do not offer any financial rewards for reporting issues to us using this method. Alternatively, you can also submit your findings through our public bug bounty program hosted by [Bugcrowd](https://bugcrowd.com/clickhouse) and be rewarded for it as per the program scope and rules of engagement. ### When Should I Report a Vulnerability? diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1dd28fa90ff..d4a3f164214 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -134,6 +134,7 @@ add_contrib (krb5-cmake krb5) add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5 add_contrib (libgsasl-cmake libgsasl) # requires krb5 add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl +add_contrib (nats-io-cmake nats-io) add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5 add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3 add_contrib (cppkafka-cmake cppkafka) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 74bbb300fa5..c289c88ef7b 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -462,5 +462,7 @@ foreach (TOOL ${PARQUET_TOOLS}) endforeach () # The library is large - avoid bloat. -target_compile_options (_arrow PRIVATE -g0) -target_compile_options (_parquet PRIVATE -g0) +if (OMIT_HEAVY_DEBUG_SYMBOLS) + target_compile_options (_arrow PRIVATE -g0) + target_compile_options (_parquet PRIVATE -g0) +endif() diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt index de6486e58fd..eabed601722 100644 --- a/contrib/aws-s3-cmake/CMakeLists.txt +++ b/contrib/aws-s3-cmake/CMakeLists.txt @@ -114,7 +114,9 @@ endif() target_link_libraries(_aws_s3 PRIVATE _aws_s3_checksums) # The library is large - avoid bloat. -target_compile_options (_aws_s3 PRIVATE -g0) -target_compile_options (_aws_s3_checksums PRIVATE -g0) +if (OMIT_HEAVY_DEBUG_SYMBOLS) + target_compile_options (_aws_s3 PRIVATE -g0) + target_compile_options (_aws_s3_checksums PRIVATE -g0) +endif() add_library(ch_contrib::aws_s3 ALIAS _aws_s3) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 761ee036e66..207b7c66371 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -171,6 +171,8 @@ target_include_directories (_curl SYSTEM PUBLIC target_link_libraries (_curl PRIVATE OpenSSL::SSL) # The library is large - avoid bloat (XXX: is it?) -target_compile_options (_curl PRIVATE -g0) +if (OMIT_HEAVY_DEBUG_SYMBOLS) + target_compile_options (_curl PRIVATE -g0) +endif() add_library (ch_contrib::curl ALIAS _curl) diff --git a/contrib/libprotobuf-mutator b/contrib/libprotobuf-mutator index ffd86a32874..a304ec48dcf 160000 --- a/contrib/libprotobuf-mutator +++ b/contrib/libprotobuf-mutator @@ -1 +1 @@ -Subproject commit ffd86a32874e5c08a143019aad1aaf0907294c9f +Subproject commit a304ec48dcf15d942607032151f7e9ee504b5dcf diff --git a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt index a623f95c418..9bbd6c17caa 100644 --- a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt +++ b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt @@ -14,8 +14,11 @@ add_library(_protobuf-mutator ${LIBRARY_DIR}/src/text_format.cc ${LIBRARY_DIR}/src/utf8_fix.cc) -target_include_directories(_protobuf-mutator BEFORE INTERFACE "${LIBRARY_DIR}") -target_include_directories(_protobuf-mutator BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") +# codegen_select_fuzzer includes ... +target_include_directories(_protobuf-mutator BEFORE PUBLIC "${LIBRARY_DIR}/src") +# ... which includes +target_include_directories(_protobuf-mutator BEFORE PUBLIC "${LIBRARY_DIR}") +target_include_directories(_protobuf-mutator BEFORE PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") target_link_libraries(_protobuf-mutator ch_contrib::protobuf) diff --git a/contrib/nats-io b/contrib/nats-io new file mode 160000 index 00000000000..6b2227f3675 --- /dev/null +++ b/contrib/nats-io @@ -0,0 +1 @@ +Subproject commit 6b2227f36757da090321e2d317569d2bd42c4cc1 diff --git a/contrib/nats-io-cmake/CMakeLists.txt b/contrib/nats-io-cmake/CMakeLists.txt new file mode 100644 index 00000000000..5588d5750c4 --- /dev/null +++ b/contrib/nats-io-cmake/CMakeLists.txt @@ -0,0 +1,59 @@ +option (ENABLE_NATS "Enable NATS" ${ENABLE_LIBRARIES}) + +if (OS_FREEBSD) + set(ENABLE_NATS OFF) + message (STATUS "Using internal nats-io library on FreeBSD is not supported") +endif() + +if (NOT ENABLE_NATS) + message(STATUS "Not using nats-io") + return() +endif() + +set(NATS_IO_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/nats-io/src") + +if(UNIX) + set(NATS_PLATFORM_INCLUDE "unix") +elseif(WIN32) + set(NATS_PLATFORM_INCLUDE "apple") +endif() + +file(GLOB PS_SOURCES "${NATS_IO_SOURCE_DIR}/${NATS_PLATFORM_INCLUDE}/*.c") +set(SRCS + "${NATS_IO_SOURCE_DIR}/asynccb.c" + "${NATS_IO_SOURCE_DIR}/buf.c" + "${NATS_IO_SOURCE_DIR}/comsock.c" + "${NATS_IO_SOURCE_DIR}/conn.c" + "${NATS_IO_SOURCE_DIR}/crypto.c" + "${NATS_IO_SOURCE_DIR}/hash.c" + "${NATS_IO_SOURCE_DIR}/js.c" + "${NATS_IO_SOURCE_DIR}/jsm.c" + "${NATS_IO_SOURCE_DIR}/kv.c" + "${NATS_IO_SOURCE_DIR}/msg.c" + "${NATS_IO_SOURCE_DIR}/nats.c" + "${NATS_IO_SOURCE_DIR}/natstime.c" + "${NATS_IO_SOURCE_DIR}/nkeys.c" + "${NATS_IO_SOURCE_DIR}/nuid.c" + "${NATS_IO_SOURCE_DIR}/opts.c" + "${NATS_IO_SOURCE_DIR}/parser.c" + "${NATS_IO_SOURCE_DIR}/pub.c" + "${NATS_IO_SOURCE_DIR}/srvpool.c" + "${NATS_IO_SOURCE_DIR}/stats.c" + "${NATS_IO_SOURCE_DIR}/status.c" + "${NATS_IO_SOURCE_DIR}/sub.c" + "${NATS_IO_SOURCE_DIR}/timer.c" + "${NATS_IO_SOURCE_DIR}/url.c" + "${NATS_IO_SOURCE_DIR}/util.c" +) + +add_library(_nats_io ${SRCS} ${PS_SOURCES}) +add_library(ch_contrib::nats_io ALIAS _nats_io) + +target_include_directories(_nats_io SYSTEM PUBLIC ${NATS_IO_SOURCE_DIR}) +target_include_directories(_nats_io SYSTEM PUBLIC ${NATS_IO_SOURCE_DIR}/adapters) +target_include_directories(_nats_io SYSTEM PUBLIC ${NATS_IO_SOURCE_DIR}/include) +target_include_directories(_nats_io SYSTEM PUBLIC ${NATS_IO_SOURCE_DIR}/${NATS_PLATFORM_INCLUDE}) + +target_link_libraries(_nats_io + PRIVATE OpenSSL::Crypto OpenSSL::SSL ch_contrib::uv +) diff --git a/contrib/poco b/contrib/poco index 0e32cb42db7..9fec8e11dbb 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 0e32cb42db76ddaa76848470219056908053b676 +Subproject commit 9fec8e11dbb6a352e1cfba8cc9e23ebd7fb77310 diff --git a/contrib/simdjson b/contrib/simdjson index de196dd7a3a..1075e8609c4 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit de196dd7a3a16e4056b0551ffa3b85c2f52581e1 +Subproject commit 1075e8609c4afa253162d441437af929c29e31bb diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 480ab3a384f..bc17105be99 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -268,10 +268,13 @@ endif() add_library (_vectorscan ${SRCS}) target_compile_options (_vectorscan PRIVATE - -g0 # library has too much debug information -fno-sanitize=undefined # assume the library takes care of itself -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # options from original build system ) +# library has too much debug information +if (OMIT_HEAVY_DEBUG_SYMBOLS) + target_compile_options (_vectorscan PRIVATE -g0) +endif() # Include version header manually generated by running the original build system target_include_directories (_vectorscan SYSTEM PRIVATE common) diff --git a/docker/server/README.md b/docker/server/README.md index c074a1bac00..2ff08620658 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -2,131 +2,138 @@ ## What is ClickHouse? -ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time. +ClickHouse is an open-source column-oriented database management system that allows the generation of analytical data reports in real-time. -ClickHouse manages extremely large volumes of data in a stable and sustainable manner. It currently powers [Yandex.Metrica](https://metrica.yandex.com/), world’s [second largest](http://w3techs.com/technologies/overview/traffic_analysis/all) web analytics platform, with over 13 trillion database records and over 20 billion events a day, generating customized reports on-the-fly, directly from non-aggregated data. This system was successfully implemented at [CERN’s LHCb experiment](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) to store and process metadata on 10bn events with over 1000 attributes per event registered in 2011. +ClickHouse manages extremely large volumes of data. It currently powers [Yandex.Metrica](https://metrica.yandex.com/), the world’s [second-largest](http://w3techs.com/technologies/overview/traffic_analysis/all) web analytics platform, with over 13 trillion database records and over 20 billion events a day, generating customized reports on-the-fly, directly from non-aggregated data. This system was successfully implemented at [CERN’s LHCb experiment](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) to store and process metadata on 10bn events with over 1000 attributes per event registered in 2011. For more information and documentation see https://clickhouse.com/. ## How to use this image ### start server instance + ```bash -$ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server +docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` -By default ClickHouse will be accessible only via docker network. See the [networking section below](#networking). +By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking). -By default, starting above server instance will be run as default user without password. +By default, starting above server instance will be run as the `default` user without a password. ### connect to it from a native client + ```bash -$ docker run -it --rm --link some-clickhouse-server:clickhouse-server --entrypoint clickhouse-client clickhouse/clickhouse-server --host clickhouse-server +docker run -it --rm --link some-clickhouse-server:clickhouse-server --entrypoint clickhouse-client clickhouse/clickhouse-server --host clickhouse-server # OR -$ docker exec -it some-clickhouse-server clickhouse-client +docker exec -it some-clickhouse-server clickhouse-client ``` -More information about [ClickHouse client](https://clickhouse.com/docs/en/interfaces/cli/). +More information about the [ClickHouse client](https://clickhouse.com/docs/en/interfaces/cli/). ### connect to it using curl ```bash echo "SELECT 'Hello, ClickHouse!'" | docker run -i --rm --link some-clickhouse-server:clickhouse-server curlimages/curl 'http://clickhouse-server:8123/?query=' -s --data-binary @- ``` + More information about [ClickHouse HTTP Interface](https://clickhouse.com/docs/en/interfaces/http/). -### stopping / removing the containter +### stopping / removing the container ```bash -$ docker stop some-clickhouse-server -$ docker rm some-clickhouse-server +docker stop some-clickhouse-server +docker rm some-clickhouse-server ``` ### networking -You can expose you ClickHouse running in docker by [mapping particular port](https://docs.docker.com/config/containers/container-networking/) from inside container to a host ports: +You can expose your ClickHouse running in docker by [mapping a particular port](https://docs.docker.com/config/containers/container-networking/) from inside the container using host ports: ```bash -$ docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server -$ echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @- +docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server +echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @- 20.12.3.3 ``` -or by allowing container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows archiving better network performance): +or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows archiving better network performance): ```bash -$ docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server -$ echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @- +docker run -d --network=host --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server +echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @- 20.12.3.3 ``` ### Volumes -Typically you may want to mount the following folders inside your container to archieve persistency: +Typically you may want to mount the following folders inside your container to achieve persistency: * `/var/lib/clickhouse/` - main folder where ClickHouse stores the data -* `/val/log/clickhouse-server/` - logs +* `/var/log/clickhouse-server/` - logs ```bash -$ docker run -d \ - -v $(realpath ./ch_data):/var/lib/clickhouse/ \ - -v $(realpath ./ch_logs):/var/log/clickhouse-server/ \ - --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server +docker run -d \ + -v $(realpath ./ch_data):/var/lib/clickhouse/ \ + -v $(realpath ./ch_logs):/var/log/clickhouse-server/ \ + --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` You may also want to mount: * `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustmenets -* `/etc/clickhouse-server/usert.d/*.xml` - files with use settings adjustmenets +* `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustmenets * `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below). ### Linux capabilities -ClickHouse has some advanced functionality which requite enabling several [linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html). +ClickHouse has some advanced functionality, which requires enabling several [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html). -It is optional and can be enabled using the following [docker command line agruments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities): +These are optional and can be enabled using the following [docker command-line arguments](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities): ```bash -$ docker run -d \ - --cap-add=SYS_NICE --cap-add=NET_ADMIN --cap-add=IPC_LOCK \ - --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server +docker run -d \ + --cap-add=SYS_NICE --cap-add=NET_ADMIN --cap-add=IPC_LOCK \ + --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` ## Configuration -Container exposes 8123 port for [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and 9000 port for [native client](https://clickhouse.com/docs/en/interfaces/tcp/). +The container exposes port 8123 for the [HTTP interface](https://clickhouse.com/docs/en/interfaces/http_interface/) and port 9000 for the [native client](https://clickhouse.com/docs/en/interfaces/tcp/). -ClickHouse configuration represented with a file "config.xml" ([documentation](https://clickhouse.com/docs/en/operations/configuration_files/)) +ClickHouse configuration is represented with a file "config.xml" ([documentation](https://clickhouse.com/docs/en/operations/configuration_files/)) ### Start server instance with custom configuration + ```bash -$ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml clickhouse/clickhouse-server +docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml clickhouse/clickhouse-server ``` -### Start server as custom user -``` +### Start server as a custom user + +```bash # $(pwd)/data/clickhouse should exist and be owned by current user -$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server +docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server ``` -When you use the image with mounting local directories inside you probably would like to not mess your directory tree with files owner and permissions. Then you could use `--user` argument. In this case, you should mount every necessary directory (`/var/lib/clickhouse` and `/var/log/clickhouse-server`) inside the container. Otherwise, image will complain and not start. + +When you use the image with local directories mounted, you probably want to specify the user to maintain the proper file ownership. Use the `--user` argument and mount `/var/lib/clickhouse` and `/var/log/clickhouse-server` inside the container. Otherwise, the image will complain and not start. ### Start server from root (useful in case of userns enabled) -``` -$ docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server + +```bash +docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" clickhouse/clickhouse-server ``` ### How to create default database and user on starting -Sometimes you may want to create user (user named `default` is used by default) and database on image starting. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`: +Sometimes you may want to create a user (user named `default` is used by default) and database on image start. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`: -``` -$ docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp clickhouse/clickhouse-server +```bash +docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp clickhouse/clickhouse-server ``` ## How to extend this image -If you would like to do additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb` it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. -Also you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization. +To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. +Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization. For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`: @@ -135,11 +142,12 @@ For example, to add an additional user and database, add the following to `/dock set -e clickhouse client -n <<-EOSQL - CREATE DATABASE docker; - CREATE TABLE docker.docker (x Int32) ENGINE = Log; + CREATE DATABASE docker; + CREATE TABLE docker.docker (x Int32) ENGINE = Log; EOSQL ``` ## License View [license information](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE) for the software contained in this image. + diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index d6bd458a01b..a124d95b360 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -63,6 +63,7 @@ RUN python3 -m pip install \ PyMySQL \ aerospike==4.0.0 \ avro==1.10.2 \ + asyncio \ cassandra-driver \ confluent-kafka==1.5.0 \ dict2xml \ @@ -75,6 +76,7 @@ RUN python3 -m pip install \ kazoo \ lz4 \ minio \ + nats-py \ protobuf \ psycopg2-binary==2.8.6 \ pymongo==3.11.0 \ diff --git a/docker/test/integration/runner/compose/docker_compose_nats.yml b/docker/test/integration/runner/compose/docker_compose_nats.yml new file mode 100644 index 00000000000..19ae4c162b1 --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_nats.yml @@ -0,0 +1,7 @@ +version: '2.3' +services: + nats1: + image: nats + ports: + - "${NATS_EXTERNAL_PORT}:${NATS_INTERNAL_PORT}" + command: "-p 4444 --user click --pass house" \ No newline at end of file diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 4f06c52a1b5..dbb90f8e537 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -164,18 +164,3 @@ ClickHouse is available in pre-built binaries and packages. Binaries are portabl They are built for stable, prestable and testing releases as long as for every commit to master and for every pull request. To find the freshest build from `master`, go to [commits page](https://github.com/ClickHouse/ClickHouse/commits/master), click on the first green check mark or red cross near commit, and click to the “Details” link right after “ClickHouse Build Check”. - -## Faster builds for development: Split build configuration {#split-build} - -Normally, ClickHouse is statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that on every change the entire binary needs to be linked, which is slow and may be inconvenient for development. There is an alternative configuration which instead creates dynamically loaded shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation: -``` --DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -``` - -Note that the split build has several drawbacks: -* There is no single `clickhouse` binary, and you have to run `clickhouse-server`, `clickhouse-client`, etc. -* Risk of segfault if you run any of the programs while rebuilding the project. -* You cannot run the integration tests since they only work a single complete binary. -* You can't easily copy the binaries elsewhere. Instead of moving a single binary you'll need to copy all binaries and libraries. - -[Original article](https://clickhouse.com/docs/en/development/build/) diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 8c1f6b5fc9e..13af1be5097 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -6,93 +6,14 @@ description: A list of third-party libraries used # Third-Party Libraries Used -The list of third-party libraries: - -| Library name | License type | -|:-|:-| -| abseil-cpp | [Apache](https://github.com/ClickHouse-Extras/abseil-cpp/blob/4f3b686f86c3ebaba7e4e926e62a79cb1c659a54/LICENSE) | -| AMQP-CPP | [Apache](https://github.com/ClickHouse-Extras/AMQP-CPP/blob/1a6c51f4ac51ac56610fa95081bd2f349911375a/LICENSE) | -| arrow | [Apache](https://github.com/ClickHouse-Extras/arrow/blob/078e21bad344747b7656ef2d7a4f7410a0a303eb/LICENSE.txt) | -| avro | [Apache](https://github.com/ClickHouse-Extras/avro/blob/e43c46e87fd32eafdc09471e95344555454c5ef8/LICENSE.txt) | -| aws | [Apache](https://github.com/ClickHouse-Extras/aws-sdk-cpp/blob/7d48b2c8193679cc4516e5bd68ae4a64b94dae7d/LICENSE.txt) | -| aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) | -| aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) | -| aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) | -| base58 | [MIT](https://github.com/ClickHouse/base-x/blob/3e58874643c087f57e82b0ff03825c933fab945a/LICENSE) | -| base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) | -| boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) | -| boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) | -| brotli | [MIT](https://github.com/google/brotli/blob/63be8a99401992075c23e99f7c84de1c653e39e2/LICENSE) | -| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/a00ccd91b3746ef2ab51d40fe3265829949d1ace/LICENSE) | -| cassandra | [Apache](https://github.com/ClickHouse-Extras/cpp-driver/blob/eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1/LICENSE.txt) | -| cctz | [Apache](https://github.com/ClickHouse-Extras/cctz/blob/c0f1bcb97fd2782f7c3f972fadd5aad5affac4b8/LICENSE.txt) | -| cityhash102 | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/cityhash102/COPYING) | -| cppkafka | [BSD 2-clause](https://github.com/mfontanini/cppkafka/blob/5a119f689f8a4d90d10a9635e7ee2bee5c127de1/LICENSE) | -| croaring | [Apache](https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/LICENSE) | -| curl | [Apache](https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/docs/LICENSE-MIXING.md) | -| cyrus-sasl | [BSD 2-clause](https://github.com/ClickHouse-Extras/cyrus-sasl/blob/e6466edfd638cc5073debe941c53345b18a09512/COPYING) | -| double-conversion | [BSD 3-clause](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) | -| dragonbox | [Apache](https://github.com/ClickHouse-Extras/dragonbox/blob/923705af6fd953aa948fc175f6020b15f7359838/LICENSE-Apache2-LLVM) | -| fast_float | [Apache](https://github.com/fastfloat/fast_float/blob/7eae925b51fd0f570ccd5c880c12e3e27a23b86f/LICENSE) | -| fastops | [MIT](https://github.com/ClickHouse-Extras/fastops/blob/88752a5e03cf34639a4a37a4b41d8b463fffd2b5/LICENSE) | -| flatbuffers | [Apache](https://github.com/ClickHouse-Extras/flatbuffers/blob/eb3f827948241ce0e701516f16cd67324802bce9/LICENSE.txt) | -| fmtlib | [Unknown](https://github.com/fmtlib/fmt/blob/c108ee1d590089ccf642fc85652b845924067af2/LICENSE.rst) | -| gcem | [Apache](https://github.com/kthohr/gcem/blob/8d4f1b5d76ea8f6ff12f3f4f34cda45424556b00/LICENSE) | -| googletest | [BSD 3-clause](https://github.com/google/googletest/blob/e7e591764baba0a0c3c9ad0014430e7a27331d16/LICENSE) | -| grpc | [Apache](https://github.com/ClickHouse-Extras/grpc/blob/60c986e15cae70aade721d26badabab1f822fdd6/LICENSE) | -| h3 | [Apache](https://github.com/ClickHouse-Extras/h3/blob/c7f46cfd71fb60e2fefc90e28abe81657deff735/LICENSE) | -| vectorscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/73695e419c27af7fe2a099c7aa57931cc02aea5d/LICENSE) | -| icu | [Public Domain](https://github.com/unicode-org/icu/blob/a56dde820dc35665a66f2e9ee8ba58e75049b668/icu4c/LICENSE) | -| icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/72d9a4a7febc904e2b0a534ccb25ae40fac5f1e5/LICENSE) | -| jemalloc | [BSD 2-clause](https://github.com/ClickHouse-Extras/jemalloc/blob/e6891d9746143bf2cf617493d880ba5a0b9a3efd/COPYING) | -| krb5 | [MIT](https://github.com/ClickHouse-Extras/krb5/blob/5149dea4e2be0f67707383d2682b897c14631374/src/lib/gssapi/LICENSE) | -| libc-headers | [LGPL](https://github.com/ClickHouse-Extras/libc-headers/blob/a720b7105a610acbd7427eea475a5b6810c151eb/LICENSE) | -| libcpuid | [BSD 2-clause](https://github.com/ClickHouse-Extras/libcpuid/blob/8db3b8d2d32d22437f063ce692a1b9bb15e42d18/COPYING) | -| libcxx | [Apache](https://github.com/ClickHouse-Extras/libcxx/blob/2fa892f69acbaa40f8a18c6484854a6183a34482/LICENSE.TXT) | -| libcxxabi | [Apache](https://github.com/ClickHouse-Extras/libcxxabi/blob/df8f1e727dbc9e2bedf2282096fa189dc3fe0076/LICENSE.TXT) | -| libdivide | [zLib](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) | -| libfarmhash | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libfarmhash/COPYING) | -| libgsasl | [LGPL](https://github.com/ClickHouse-Extras/libgsasl/blob/383ee28e82f69fa16ed43b48bd9c8ee5b313ab84/LICENSE) | -| libhdfs3 | [Apache](https://github.com/ClickHouse-Extras/libhdfs3/blob/095b9d48b400abb72d967cb0539af13b1e3d90cf/LICENSE.txt) | -| libmetrohash | [Apache](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) | -| libpq | [Unknown](https://github.com/ClickHouse-Extras/libpq/blob/e071ea570f8985aa00e34f5b9d50a3cfe666327e/COPYRIGHT) | -| libpqxx | [BSD 3-clause](https://github.com/ClickHouse-Extras/libpqxx/blob/357608d11b7a1961c3fb7db2ef9a5dbb2e87da77/COPYING) | -| librdkafka | [MIT](https://github.com/ClickHouse-Extras/librdkafka/blob/b8554f1682062c85ba519eb54ef2f90e02b812cb/LICENSE.murmur2) | -| libunwind | [Apache](https://github.com/ClickHouse-Extras/libunwind/blob/6b816d2fba3991f8fd6aaec17d92f68947eab667/LICENSE.TXT) | -| libuv | [BSD](https://github.com/ClickHouse-Extras/libuv/blob/e2e9b7e9f978ce8a1367b5fe781d97d1ce9f94ab/LICENSE) | -| llvm | [Apache](https://github.com/ClickHouse-Extras/llvm/blob/e5751459412bce1391fb7a2e9bbc01e131bf72f1/llvm/LICENSE.TXT) | -| lz4 | [BSD](https://github.com/lz4/lz4/blob/f39b79fb02962a1cd880bbdecb6dffba4f754a11/LICENSE) | -| mariadb-connector-c | [LGPL](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/5f4034a3a6376416504f17186c55fe401c6d8e5e/COPYING.LIB) | -| miniselect | [Boost](https://github.com/danlark1/miniselect/blob/be0af6bd0b6eb044d1acc4f754b229972d99903a/LICENSE_1_0.txt) | -| msgpack-c | [Boost](https://github.com/msgpack/msgpack-c/blob/46684265d50b5d1b062d4c5c428ba08462844b1d/LICENSE_1_0.txt) | -| murmurhash | [Public Domain](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) | -| NuRaft | [Apache](https://github.com/ClickHouse-Extras/NuRaft/blob/7ecb16844af6a9c283ad432d85ecc2e7d1544676/LICENSE) | -| openldap | [Unknown](https://github.com/ClickHouse-Extras/openldap/blob/0208811b6043ca06fda8631a5e473df1ec515ccb/LICENSE) | -| orc | [Apache](https://github.com/ClickHouse-Extras/orc/blob/0a936f6bbdb9303308973073f8623b5a8d82eae1/LICENSE) | -| poco | [Boost](https://github.com/ClickHouse-Extras/poco/blob/7351c4691b5d401f59e3959adfc5b4fa263b32da/LICENSE) | -| protobuf | [BSD 3-clause](https://github.com/ClickHouse-Extras/protobuf/blob/75601841d172c73ae6bf4ce8121f42b875cdbabd/LICENSE) | -| rapidjson | [MIT](https://github.com/ClickHouse-Extras/rapidjson/blob/c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa/bin/jsonschema/LICENSE) | -| re2 | [BSD 3-clause](https://github.com/google/re2/blob/13ebb377c6ad763ca61d12dd6f88b1126bd0b911/LICENSE) | -| replxx | [BSD 3-clause](https://github.com/ClickHouse-Extras/replxx/blob/c81be6c68b146f15f2096b7ef80e3f21fe27004c/LICENSE.md) | -| rocksdb | [BSD 3-clause](https://github.com/ClickHouse-Extras/rocksdb/blob/b6480c69bf3ab6e298e0d019a07fd4f69029b26a/LICENSE.leveldb) | -| s2geometry | [Apache](https://github.com/ClickHouse-Extras/s2geometry/blob/20ea540d81f4575a3fc0aea585aac611bcd03ede/LICENSE) | -| sentry-native | [MIT](https://github.com/ClickHouse-Extras/sentry-native/blob/94644e92f0a3ff14bd35ed902a8622a2d15f7be4/LICENSE) | -| simdjson | [Apache](https://github.com/simdjson/simdjson/blob/8df32cea3359cb30120795da6020b3b73da01d38/LICENSE) | -| snappy | [Public Domain](https://github.com/google/snappy/blob/3f194acb57e0487531c96b97af61dcbd025a78a3/COPYING) | -| sparsehash-c11 | [BSD 3-clause](https://github.com/sparsehash/sparsehash-c11/blob/cf0bffaa456f23bc4174462a789b90f8b6f5f42f/LICENSE) | -| stats | [Apache](https://github.com/kthohr/stats/blob/b6dd459c10a88c7ea04693c007e9e35820c5d9ad/LICENSE) | -| thrift | [Apache](https://github.com/apache/thrift/blob/010ccf0a0c7023fea0f6bf4e4078ebdff7e61982/LICENSE) | -| unixodbc | [LGPL](https://github.com/ClickHouse-Extras/UnixODBC/blob/b0ad30f7f6289c12b76f04bfb9d466374bb32168/COPYING) | -| xz | [Public Domain](https://github.com/xz-mirror/xz/blob/869b9d1b4edd6df07f819d360d306251f8147353/COPYING) | -| zlib-ng | [zLib](https://github.com/ClickHouse-Extras/zlib-ng/blob/6a5e93b9007782115f7f7e5235dedc81c4f1facb/LICENSE.md) | -| zstd | [BSD](https://github.com/facebook/zstd/blob/a488ba114ec17ea1054b9057c26a046fc122b3b6/LICENSE) | - -The list of third-party libraries can be obtained by the following query: +ClickHouse utilizes third-party libraries for different purposes, e.g., to connect to other databases, to decode (encode) data during load (save) from (to) disk or to implement certain specialized SQL functions. To be independent of the available libraries in the target system, each third-party library is imported as a Git submodule into ClickHouse's source tree and compiled and linked with ClickHouse. A list of third-party libraries and their licenses can be obtained by the following query: ``` sql SELECT library_name, license_type, license_path FROM system.licenses ORDER BY library_name COLLATE 'en'; ``` +(Note that the listed libraries are the ones located in the `contrib/` directory of the ClickHouse repository. Depending on the build options, some of of the libraries may have not been compiled, and as a result, their functionality may not be available at runtime. + [Example](https://play.clickhouse.com/play?user=play#U0VMRUNUIGxpYnJhcnlfbmFtZSwgbGljZW5zZV90eXBlLCBsaWNlbnNlX3BhdGggRlJPTSBzeXN0ZW0ubGljZW5zZXMgT1JERVIgQlkgbGlicmFyeV9uYW1lIENPTExBVEUgJ2VuJw==) ## Adding new third-party libraries and maintaining patches in third-party libraries {#adding-third-party-libraries} diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 38d99430193..77ddae6a756 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -276,3 +276,23 @@ Testing will commence as soon as ClickHouse employees label your PR with a tag The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the “Details” link next to “ClickHouse build check” entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear). Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways. + +## Faster builds for development: Split build configuration {#split-build} + +ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation: +``` +-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 +``` + +Note that the split build has several drawbacks: +* There is no single `clickhouse` binary, and you have to run `clickhouse-server`, `clickhouse-client`, etc. +* Risk of segfault if you run any of the programs while rebuilding the project. +* You cannot run the integration tests since they only work a single complete binary. +* You can't easily copy the binaries elsewhere. Instead of moving a single binary you'll need to copy all binaries and libraries. + +If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options +``` +-DENABLE_LIBRARIES=0 -DENABLE_EMBEDDED_COMPILER=0 +``` + +In case of problems with any of the development options, you are on your own! diff --git a/docs/en/engines/table-engines/integrations/nats.md b/docs/en/engines/table-engines/integrations/nats.md new file mode 100644 index 00000000000..7c975653f0e --- /dev/null +++ b/docs/en/engines/table-engines/integrations/nats.md @@ -0,0 +1,163 @@ +--- +sidebar_position: 14 +sidebar_label: NATS +--- + +# NATS Engine {#redisstreams-engine} + +This engine allows integrating ClickHouse with [NATS](https://nats.io/). + +`NATS` lets you: + +- Publish or subcribe to message subjects. +- Process new messages as they become available. + +## Creating a Table {#table_engine-redisstreams-creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = NATS SETTINGS + nats_url = 'host:port', + nats_subjects = 'subject1,subject2,...', + nats_format = 'data_format'[,] + [nats_row_delimiter = 'delimiter_symbol',] + [nats_schema = '',] + [nats_num_consumers = N,] + [nats_queue_group = 'group_name',] + [nats_secure = false,] + [nats_max_reconnect = N,] + [nats_reconnect_wait = N,] + [nats_server_list = 'host1:port1,host2:port2,...',] + [nats_skip_broken_messages = N,] + [nats_max_block_size = N,] + [nats_flush_interval_ms = N,] + [nats_username = 'user',] + [nats_password = 'password'] + [redis_password = 'clickhouse'] +``` + +Required parameters: + +- `nats_url` – host:port (for example, `localhost:5672`).. +- `nats_subjects` – List of subject for NATS table to subscribe/publsh to. Supports wildcard subjects like `foo.*.bar` or `baz.>` +- `nats_format` – Message format. Uses the same notation as the SQL `FORMAT` function, such as `JSONEachRow`. For more information, see the [Formats](../../../interfaces/formats.md) section. + +Optional parameters: + +- `nats_row_delimiter` – Delimiter character, which ends the message. +- `nats_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. +- `nats_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. +- `nats_queue_group` – Name for queue group of NATS subscribers. Default is the table name. +- `nats_max_reconnect` – Maximum amount of reconnection attempts per try to connect to NATS. Default: `5`. +- `nats_reconnect_wait` – Amount of time in milliseconds to sleep between each reconnect attempt. Default: `5000`. +- `nats_server_list` - Server list for connection. Can be specified to connect to NATS cluster. +- `nats_skip_broken_messages` - NATS message parser tolerance to schema-incompatible messages per block. Default: `0`. If `nats_skip_broken_messages = N` then the engine skips *N* RabbitMQ messages that cannot be parsed (a message equals a row of data). +- `nats_max_block_size` - Number of row collected by poll(s) for flushing data from NATS. +- `nats_flush_interval_ms` - Timeout for flushing data read from NATS. +- `nats_username` - NATS username. +- `nats_password` - NATS password. +- `nats_token` - NATS auth token. + +SSL connection: + +For secure connection use `nats_secure = 1`. +The default behaviour of the used library is not to check if the created TLS connection is sufficiently secure. Whether the certificate is expired, self-signed, missing or invalid: the connection is simply permitted. More strict checking of certificates can possibly be implemented in the future. + +Writing to NATS table: + +If table reads only from one subject, any insert will publish to the same subject. +However, if table reads from multiple subjects, we need to specify which subject we want to publish to. +That is why whenever inserting into table with multiple subjects, setting `stream_like_engine_insert_queue` is needed. +You can select one of the subjects the table reads from and publish your data there. For example: + +``` sql + CREATE TABLE queue ( + key UInt64, + value UInt64 + ) ENGINE = NATS + SETTINGS nats_url = 'localhost:4444', + nats_subjects = 'subject1,subject2', + nats_format = 'JSONEachRow'; + + INSERT INTO queue + SETTINGS stream_like_engine_insert_queue = 'subject2' + VALUES (1, 1); +``` + +Also format settings can be added along with nats-related settings. + +Example: + +``` sql + CREATE TABLE queue ( + key UInt64, + value UInt64, + date DateTime + ) ENGINE = NATS + SETTINGS nats_url = 'localhost:4444', + nats_subjects = 'subject1', + nats_format = 'JSONEachRow', + date_time_input_format = 'best_effort'; +``` + +The NATS server configuration can be added using the ClickHouse config file. + More specifically you can add Redis password for NATS engine: + +``` xml + + click + house + clickhouse + +``` + +## Description {#description} + +`SELECT` is not particularly useful for reading messages (except for debugging), because each message can be read only once. It is more practical to create real-time threads using [materialized views](../../../sql-reference/statements/create/view.md). To do this: + +1. Use the engine to create a NATS consumer and consider it a data stream. +2. Create a table with the desired structure. +3. Create a materialized view that converts data from the engine and puts it into a previously created table. + +When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. This allows you to continually receive messages from NATS and convert them to the required format using `SELECT`. +One NATS table can have as many materialized views as you like, they do not read data from the table directly, but receive new records (in blocks), this way you can write to several tables with different detail level (with grouping - aggregation and without). + +Example: + +``` sql + CREATE TABLE queue ( + key UInt64, + value UInt64 + ) ENGINE = NATS + SETTINGS nats_url = 'localhost:4444', + nats_subjects = 'subject1', + nats_format = 'JSONEachRow', + date_time_input_format = 'best_effort'; + + CREATE TABLE daily (key UInt64, value UInt64) + ENGINE = MergeTree() ORDER BY key; + + CREATE MATERIALIZED VIEW consumer TO daily + AS SELECT key, value FROM queue; + + SELECT key, value FROM daily ORDER BY key; +``` + +To stop receiving streams data or to change the conversion logic, detach the materialized view: + +``` sql + DETACH TABLE consumer; + ATTACH TABLE consumer; +``` + +If you want to change the target table by using `ALTER`, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view. + +## Virtual Columns {#virtual-columns} + +- `_subject` - NATS message subject. + +[Original article](https://clickhouse.com/docs/en/engines/table-engines/integrations/nats/) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 00fa382fd4d..5d8ed9cdacd 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -11,68 +11,69 @@ results of a `SELECT`, and to perform `INSERT`s into a file-backed table. The supported formats are: | Format | Input | Output | -|-------------------------------------------------------------------------------------------|-------|--------| -| [TabSeparated](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | -| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | -| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | -| [TabSeparatedRawWithNames](#tabseparatedrawwithnames) | ✔ | ✔ | -| [TabSeparatedRawWithNamesAndTypes](#tabseparatedrawwithnamesandtypes) | ✔ | ✔ | -| [Template](#format-template) | ✔ | ✔ | -| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | -| [CSV](#csv) | ✔ | ✔ | -| [CSVWithNames](#csvwithnames) | ✔ | ✔ | -| [CSVWithNamesAndTypes](#csvwithnamesandtypes) | ✔ | ✔ | -| [CustomSeparated](#format-customseparated) | ✔ | ✔ | -| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ | -| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ | -| [Values](#data-format-values) | ✔ | ✔ | -| [Vertical](#vertical) | ✗ | ✔ | -| [JSON](#json) | ✗ | ✔ | -| [JSONAsString](#jsonasstring) | ✔ | ✗ | -| [JSONStrings](#jsonstrings) | ✗ | ✔ | -| [JSONColumns](#jsoncolumns) | ✔ | ✔ | -| [JSONColumnsWithMetadata](#jsoncolumnswithmetadata) | ✗ | ✔ | -| [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | -| [JSONCompactColumns](#jsoncompactcolumns) | ✔ | ✔ | -| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | -| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | -| [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ | -| [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ | -| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | -| [JSONCompactEachRowWithNames](#jsoncompacteachrowwithnames) | ✔ | ✔ | -| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | -| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ | -| [JSONCompactStringsEachRowWithNames](#jsoncompactstringseachrowwithnames) | ✔ | ✔ | -| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ | -| [TSKV](#tskv) | ✔ | ✔ | -| [Pretty](#pretty) | ✗ | ✔ | -| [PrettyCompact](#prettycompact) | ✗ | ✔ | -| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | -| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | -| [PrettySpace](#prettyspace) | ✗ | ✔ | -| [Prometheus](#prometheus) | ✗ | ✔ | -| [Protobuf](#protobuf) | ✔ | ✔ | -| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | -| [Avro](#data-format-avro) | ✔ | ✔ | -| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | -| [Parquet](#data-format-parquet) | ✔ | ✔ | -| [Arrow](#data-format-arrow) | ✔ | ✔ | -| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✔ | -| [RowBinary](#rowbinary) | ✔ | ✔ | -| [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [Native](#native) | ✔ | ✔ | -| [Null](#null) | ✗ | ✔ | -| [XML](#xml) | ✗ | ✔ | -| [CapnProto](#capnproto) | ✔ | ✔ | -| [LineAsString](#lineasstring) | ✔ | ✗ | -| [Regexp](#data-format-regexp) | ✔ | ✗ | -| [RawBLOB](#rawblob) | ✔ | ✔ | -| [MsgPack](#msgpack) | ✔ | ✔ | -| [MySQLDump](#mysqldump) | ✔ | ✗ | +|-------------------------------------------------------------------------------------------|------|--------| +| [TabSeparated](#tabseparated) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | +| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | +| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | +| [TabSeparatedRawWithNames](#tabseparatedrawwithnames) | ✔ | ✔ | +| [TabSeparatedRawWithNamesAndTypes](#tabseparatedrawwithnamesandtypes) | ✔ | ✔ | +| [Template](#format-template) | ✔ | ✔ | +| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | +| [CSV](#csv) | ✔ | ✔ | +| [CSVWithNames](#csvwithnames) | ✔ | ✔ | +| [CSVWithNamesAndTypes](#csvwithnamesandtypes) | ✔ | ✔ | +| [CustomSeparated](#format-customseparated) | ✔ | ✔ | +| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ | +| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ | +| [SQLInsert](#sqlinsert) | ✗ | ✔ | +| [Values](#data-format-values) | ✔ | ✔ | +| [Vertical](#vertical) | ✗ | ✔ | +| [JSON](#json) | ✗ | ✔ | +| [JSONAsString](#jsonasstring) | ✔ | ✗ | +| [JSONStrings](#jsonstrings) | ✗ | ✔ | +| [JSONColumns](#jsoncolumns) | ✔ | ✔ | +| [JSONColumnsWithMetadata](#jsoncolumnswithmetadata) | ✗ | ✔ | +| [JSONCompact](#jsoncompact) | ✗ | ✔ | +| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | +| [JSONCompactColumns](#jsoncompactcolumns) | ✔ | ✔ | +| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | +| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | +| [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ | +| [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ | +| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | +| [JSONCompactEachRowWithNames](#jsoncompacteachrowwithnames) | ✔ | ✔ | +| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ | +| [JSONCompactStringsEachRowWithNames](#jsoncompactstringseachrowwithnames) | ✔ | ✔ | +| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ | +| [TSKV](#tskv) | ✔ | ✔ | +| [Pretty](#pretty) | ✗ | ✔ | +| [PrettyCompact](#prettycompact) | ✗ | ✔ | +| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | +| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | +| [PrettySpace](#prettyspace) | ✗ | ✔ | +| [Prometheus](#prometheus) | ✗ | ✔ | +| [Protobuf](#protobuf) | ✔ | ✔ | +| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | +| [Avro](#data-format-avro) | ✔ | ✔ | +| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | +| [Parquet](#data-format-parquet) | ✔ | ✔ | +| [Arrow](#data-format-arrow) | ✔ | ✔ | +| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✔ | +| [RowBinary](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [Native](#native) | ✔ | ✔ | +| [Null](#null) | ✗ | ✔ | +| [XML](#xml) | ✗ | ✔ | +| [CapnProto](#capnproto) | ✔ | ✔ | +| [LineAsString](#lineasstring) | ✔ | ✗ | +| [Regexp](#data-format-regexp) | ✔ | ✗ | +| [RawBLOB](#rawblob) | ✔ | ✔ | +| [MsgPack](#msgpack) | ✔ | ✔ | +| [MySQLDump](#mysqldump) | ✔ | ✗ | You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section. @@ -468,6 +469,34 @@ Also prints the header row with column names, similar to [TabSeparatedWithNames] Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). +## SQLInsert {#sqlinsert} + +Outputs data as a sequence of `INSERT INTO table (columns...) VALUES (...), (...) ...;` statements. + +Example: + +```sql +SELECT number AS x, number + 1 AS y, 'Hello' AS z FROM numbers(10) FORMAT SQLInsert SETTINGS output_format_sql_insert_max_batch_size = 2 +``` + +```sql +INSERT INTO table (x, y, z) VALUES (0, 1, 'Hello'), (1, 2, 'Hello'); +INSERT INTO table (x, y, z) VALUES (2, 3, 'Hello'), (3, 4, 'Hello'); +INSERT INTO table (x, y, z) VALUES (4, 5, 'Hello'), (5, 6, 'Hello'); +INSERT INTO table (x, y, z) VALUES (6, 7, 'Hello'), (7, 8, 'Hello'); +INSERT INTO table (x, y, z) VALUES (8, 9, 'Hello'), (9, 10, 'Hello'); +``` + +To read data output by this format ypu can use [MySQLDump](#mysqldump) input format. + +### SQLInsert format settings {#sqlinsert-format-settings} + +- [output_format_sql_insert_max_batch_size](../operations/settings/settings.md#output_format_sql_insert_max_batch_size) - The maximum number of rows in one INSERT statement. Default value - `65505`. +- [output_format_sql_insert_table_name](../operations/settings/settings.md#output_format_sql_insert_table_name) - The name of table in the output INSERT query. Default value - `'table'`. +- [output_format_sql_insert_include_column_names](../operations/settings/settings.md#output_format_sql_insert_include_column_names) - Include column names in INSERT query. Default value - `true`. +- [output_format_sql_insert_use_replace](../operations/settings/settings.md#output_format_sql_insert_use_replace) - Use REPLACE statement instead of INSERT. Default value - `false`. +- [output_format_sql_insert_quote_names](../operations/settings/settings.md#output_format_sql_insert_quote_names) - Quote column names with "\`" characters . Default value - `true`. + ## JSON {#json} Outputs data in JSON format. Besides data tables, it also outputs column names and types, along with some additional information: the total number of output rows, and the number of rows that could have been output if there weren’t a LIMIT. Example: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 85265448c03..75c2aa57b32 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4637,3 +4637,35 @@ Possible values: - 1 — Enabled. Default value: 1. + +## SQLInsert format settings {$sqlinsert-format-settings} + +### output_format_sql_insert_max_batch_size {#output_format_sql_insert_max_batch_size} + +The maximum number of rows in one INSERT statement. + +Default value: `65505`. + +### output_format_sql_insert_table_name {#output_format_sql_insert_table_name} + +The name of table that will be used in the output INSERT statement. + +Default value: `'table''`. + +### output_format_sql_insert_include_column_names {#output_format_sql_insert_include_column_names} + +Include column names in INSERT statement. + +Default value: `true`. + +### output_format_sql_insert_use_replace {#output_format_sql_insert_use_replace} + +Use REPLACE keyword instead of INSERT. + +Default value: `false`. + +### output_format_sql_insert_quote_names {#output_format_sql_insert_quote_names} + +Quote column names with "`" characters + +Default value: `true`. diff --git a/docs/en/sql-reference/statements/create/function.md b/docs/en/sql-reference/statements/create/function.md index 7d9a727a70d..0a452b6c4d2 100644 --- a/docs/en/sql-reference/statements/create/function.md +++ b/docs/en/sql-reference/statements/create/function.md @@ -10,7 +10,7 @@ Creates a user defined function from a lambda expression. The expression must co **Syntax** ```sql -CREATE FUNCTION name AS (parameter0, ...) -> expression +CREATE FUNCTION name [ON CLUSTER cluster] AS (parameter0, ...) -> expression ``` A function can have an arbitrary number of parameters. diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 49862cbfc02..9621cd4944f 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -105,7 +105,7 @@ System functions can not be dropped. **Syntax** ``` sql -DROP FUNCTION [IF EXISTS] function_name +DROP FUNCTION [IF EXISTS] function_name [on CLUSTER cluster] ``` **Example** diff --git a/docs/zh/faq/use-cases/key-value.md b/docs/zh/faq/use-cases/key-value.md index 4a611512586..79c221d30c8 100644 --- a/docs/zh/faq/use-cases/key-value.md +++ b/docs/zh/faq/use-cases/key-value.md @@ -5,12 +5,12 @@ sidebar_position: 101 --- # 我能把 ClickHouse 当做Key-value 键值存储来使用吗？ {#can-i-use-clickhouse-as-a-key-value-storage}. -简短的回答是 **不能** 。关键值的工作量是在列表中的最高位置时，**不能**{.text-danger}使用ClickHouse的情况。它是一个[OLAP](../../faq/general/olap.md)系统，毕竟有很多优秀的键值存储系统在那里。 +简短的回答是 **不能** 。键值类型负载是ClickHouse最**不适合**的多种场景之一。ClickHouse 毕竟只是一个[OLAP](../../faq/general/olap.md)系统，对于这类负载来说，目前还是有很多优秀的键值存储系统可供选择。 -然而，可能在某些情况下，使用ClickHouse进行类似键值的查询仍然是有意义的。通常，是一些低预算的产品，主要的工作负载是分析性的，很适合ClickHouse，但也有一些次要的过程需要一个键值模式，请求吞吐量不是很高，没有严格的延迟要求。如果你有无限的预算，你会为这样的次要工作负载安装一个次要的键值数据库，但实际上，多维护一个存储系统（监控、备份等）会有额外的成本，这可能是值得避免的。 +然而，在某些情况下，使用ClickHouse进行类似键值的查询仍然是有意义的。通常，一些主要的工作负载是分析性的比较适合使用Clickhouse低预算的产品中，也有一些次要的操作是需要使用键值模式的，同时这些操作的请求吞吐量不会很高，也没有严格的延迟要求。如果你有无限的预算，你会为这样的次要工作负载安装一个次要的键值数据库，但实际上，多维护一个存储系统（监控、备份等）会有额外的成本，这是可以考虑避免的。 -如果你决定违背建议，对ClickHouse运行一些类似键值的查询，这里有一些提示。 +如果你决定不遵从这些建议，想要使用ClickHouse运行一些类似键值的查询，那么这里有一些提示。 -- ClickHouse中点查询昂贵的关键原因是其稀疏的主索引[MergeTree表引擎家族]（../../engines/table-engines/mergetree-family/mergetree.md）。这个索引不能指向每一行具体的数据，相反，它指向每N行，系统必须从邻近的N行扫描到所需的行，沿途读取过多的数据。在一个键值场景中，通过`index_granularity`的设置来减少N的值可能是有用的。 -- ClickHouse将每一列保存在一组单独的文件中，所以要组装一个完整的行，它需要通过这些文件中的每一个。它们的数量随着列数的增加而线性增加，所以在键值场景中，可能值得避免使用许多列，并将所有的有效数据放在一个单一的`String`列中，并以某种序列化格式（如JSON、Protobuf或任何有效的格式）进行编码。 +- ClickHouse中点查询开销大的关键原因是MergeTree表引擎家族[MergeTree表引擎家族]（../../engines/table-engines/mergetree-family/mergetree.md）采用的稀疏主索引。这个索引不能指向每一行具体的数据，相反，它指向每N行，系统必须从邻近的N行扫描到所需的行，沿途读取过多的数据。在一个键值场景中，通过`index_granularity`的设置来减少N的值可能是有用的。 +- ClickHouse将每一列保存在一组单独的文件中，所以要组装一个完整的行，它需要访问文件组中的每一个文件。访问数据数量会随着列数的增加而线性增加，所以在键值场景中，需要避免使用许多列，并将所有的有效数据放在一个单一的`String`列中，并以某种序列化格式（如JSON、Protobuf或任何有效的格式）进行编码。 - 还有一种方法，使用[Join](../../engines/table-engines/special/join.md)表引擎代替正常的`MergeTree`表和[joinGet](../../sql-reference/functions/other-functions.md#joinget) 函数来检索数据。它可以提供更好的查询性能，但可能有一些可用性和可靠性问题。下面是一个[使用实例](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00800_versatile_storage_join.sql#L49-L51)。 diff --git a/programs/su/clickhouse-su.cpp b/programs/su/clickhouse-su.cpp index 9aa41085094..0979abf353d 100644 --- a/programs/su/clickhouse-su.cpp +++ b/programs/su/clickhouse-su.cpp @@ -59,7 +59,7 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid) throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR); if (!result) - throw Exception("Group {} is not found in the system", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid); gid = entry.gr_gid; } @@ -84,7 +84,7 @@ void setUserAndGroup(std::string arg_uid, std::string arg_gid) throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR); if (!result) - throw Exception("User {} is not found in the system", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid); uid = entry.pw_uid; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 10bdc464ac6..deb3206db31 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,6 +96,10 @@ if (TARGET ch_contrib::rdkafka) add_headers_and_sources(dbms Storages/Kafka) endif() +if (TARGET ch_contrib::nats_io) + add_headers_and_sources(dbms Storages/NATS) +endif() + add_headers_and_sources(dbms Storages/MeiliSearch) if (TARGET ch_contrib::amqp_cpp) @@ -371,6 +375,10 @@ if (TARGET ch_contrib::rdkafka) dbms_target_link_libraries(PRIVATE ch_contrib::rdkafka ch_contrib::cppkafka) endif() +if (TARGET ch_contrib::nats_io) + dbms_target_link_libraries(PRIVATE ch_contrib::nats_io) +endif() + if (TARGET ch_contrib::sasl2) dbms_target_link_libraries(PRIVATE ch_contrib::sasl2) endif() diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 396fd97368e..8230c97f49c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1834,9 +1834,21 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) bool ClientBase::processQueryText(const String & text) { - if (exit_strings.end() != exit_strings.find(trim(text, [](char c) { return isWhitespaceASCII(c) || c == ';'; }))) + auto trimmed_input = trim(text, [](char c) { return isWhitespaceASCII(c) || c == ';'; }); + + if (exit_strings.end() != exit_strings.find(trimmed_input)) return false; + if (trimmed_input.starts_with("\\i")) + { + size_t skip_prefix_size = std::strlen("\\i"); + auto file_name = trim( + trimmed_input.substr(skip_prefix_size, trimmed_input.size() - skip_prefix_size), + [](char c) { return isWhitespaceASCII(c); }); + + return processMultiQueryFromFile(file_name); + } + if (!is_multiquery) { assert(!query_fuzzer_runs); @@ -2019,6 +2031,17 @@ void ClientBase::runInteractive() } +bool ClientBase::processMultiQueryFromFile(const String & file_name) +{ + String queries_from_file; + + ReadBufferFromFile in(file_name); + readStringUntilEOF(queries_from_file, in); + + return executeMultiQuery(queries_from_file); +} + + void ClientBase::runNonInteractive() { if (delayed_interactive) @@ -2026,23 +2049,13 @@ void ClientBase::runNonInteractive() if (!queries_files.empty()) { - auto process_multi_query_from_file = [&](const String & file) - { - String queries_from_file; - - ReadBufferFromFile in(file); - readStringUntilEOF(queries_from_file, in); - - return executeMultiQuery(queries_from_file); - }; - for (const auto & queries_file : queries_files) { for (const auto & interleave_file : interleave_queries_files) - if (!process_multi_query_from_file(interleave_file)) + if (!processMultiQueryFromFile(interleave_file)) return; - if (!process_multi_query_from_file(queries_file)) + if (!processMultiQueryFromFile(queries_file)) return; } diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index d34fe282839..ec2267a3be6 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -154,6 +154,7 @@ private: protected: static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context); + bool processMultiQueryFromFile(const String & file_name); bool is_interactive = false; /// Use either interactive line editing interface or batch mode. bool is_multiquery = false; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 435e6bf1fbc..d8e98ec9406 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -715,29 +715,37 @@ ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const template -void ColumnNullable::applyNullMapImpl(const ColumnUInt8 & map) +void ColumnNullable::applyNullMapImpl(const NullMap & map) { - NullMap & arr1 = getNullMapData(); - const NullMap & arr2 = map.getData(); + NullMap & arr = getNullMapData(); - if (arr1.size() != arr2.size()) + if (arr.size() != map.size()) throw Exception{"Inconsistent sizes of ColumnNullable objects", ErrorCodes::LOGICAL_ERROR}; - for (size_t i = 0, size = arr1.size(); i < size; ++i) - arr1[i] |= negative ^ arr2[i]; + for (size_t i = 0, size = arr.size(); i < size; ++i) + arr[i] |= negative ^ map[i]; } - -void ColumnNullable::applyNullMap(const ColumnUInt8 & map) +void ColumnNullable::applyNullMap(const NullMap & map) { applyNullMapImpl(map); } -void ColumnNullable::applyNegatedNullMap(const ColumnUInt8 & map) +void ColumnNullable::applyNullMap(const ColumnUInt8 & map) +{ + applyNullMapImpl(map.getData()); +} + +void ColumnNullable::applyNegatedNullMap(const NullMap & map) { applyNullMapImpl(map); } +void ColumnNullable::applyNegatedNullMap(const ColumnUInt8 & map) +{ + applyNullMapImpl(map.getData()); +} + void ColumnNullable::applyNullMap(const ColumnNullable & other) { diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 60951dfcc2e..52e57f7f0d0 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -199,7 +199,9 @@ public: /// columns. void applyNullMap(const ColumnNullable & other); void applyNullMap(const ColumnUInt8 & map); + void applyNullMap(const NullMap & map); void applyNegatedNullMap(const ColumnUInt8 & map); + void applyNegatedNullMap(const NullMap & map); /// Check that size of null map equals to size of nested column. void checkConsistency() const; @@ -209,7 +211,7 @@ private: WrappedPtr null_map; template - void applyNullMapImpl(const ColumnUInt8 & map); + void applyNullMapImpl(const NullMap & map); int compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint, const Collator * collator=nullptr) const; diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp index d2c16c186ce..9b2589f0589 100644 --- a/src/Common/Epoll.cpp +++ b/src/Common/Epoll.cpp @@ -70,6 +70,9 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, bool blocki if (ready_size == -1 && errno != EINTR) throwFromErrno("Error in epoll_wait", DB::ErrorCodes::EPOLL_ERROR); + + if (errno == EINTR) + LOG_TEST(&Poco::Logger::get("Epoll"), "EINTR"); } while (ready_size <= 0 && (ready_size != 0 || blocking)); diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 8e7eaf4c6e6..206f2061cde 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -633,6 +633,7 @@ M(662, FS_METADATA_ERROR) \ M(663, INCONSISTENT_METADATA_FOR_BACKUP) \ M(664, ACCESS_STORAGE_DOESNT_ALLOW_BACKUP) \ + M(665, CANNOT_CONNECT_NATS) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp index a7c74dab8be..1301ebce0ba 100644 --- a/src/Common/TimerDescriptor.cpp +++ b/src/Common/TimerDescriptor.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -70,6 +72,8 @@ void TimerDescriptor::drain() const if (errno != EINTR) throwFromErrno("Cannot drain timer_fd", ErrorCodes::CANNOT_READ_FROM_SOCKET); + else + LOG_TEST(&Poco::Logger::get("TimerDescriptor"), "EINTR"); } } } diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index 73c7da25a8b..79f9943cb57 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -281,6 +281,13 @@ struct SetResponse : virtual Response size_t bytesSize() const override { return sizeof(stat); } }; +enum class ListRequestType : uint8_t +{ + ALL, + PERSISTENT_ONLY, + EPHEMERAL_ONLY +}; + struct ListRequest : virtual Request { String path; @@ -492,6 +499,7 @@ public: virtual void list( const String & path, + ListRequestType list_request_type, ListCallback callback, WatchCallback watch) = 0; diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 3d2d5fcb667..3af5dfcc177 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -1,3 +1,4 @@ +#include "Common/ZooKeeper/IKeeper.h" #include #include #include @@ -119,12 +120,17 @@ struct TestKeeperSetRequest final : SetRequest, TestKeeperRequest } }; -struct TestKeeperListRequest final : ListRequest, TestKeeperRequest +struct TestKeeperListRequest : ListRequest, TestKeeperRequest { ResponsePtr createResponse() const override; std::pair process(TestKeeper::Container & container, int64_t zxid) const override; }; +struct TestKeeperFilteredListRequest final : TestKeeperListRequest +{ + ListRequestType list_request_type; +}; + struct TestKeeperCheckRequest final : CheckRequest, TestKeeperRequest { TestKeeperCheckRequest() = default; @@ -390,8 +396,18 @@ std::pair TestKeeperListRequest::process(TestKeeper::Containe child_it != container.end() && startsWith(child_it->first, path_prefix); ++child_it) { + using enum ListRequestType; if (parentPath(child_it->first) == path) - response.names.emplace_back(baseName(child_it->first)); + { + ListRequestType list_request_type = ALL; + if (const auto * filtered_list = dynamic_cast(this)) + list_request_type = filtered_list->list_request_type; + + const auto is_ephemeral = child_it->second.stat.ephemeralOwner != 0; + if (list_request_type == ALL || (is_ephemeral && list_request_type == EPHEMERAL_ONLY) + || (!is_ephemeral && list_request_type == PERSISTENT_ONLY)) + response.names.emplace_back(baseName(child_it->first)); + } } response.stat = it->second.stat; @@ -768,11 +784,13 @@ void TestKeeper::set( void TestKeeper::list( const String & path, + ListRequestType list_request_type, ListCallback callback, WatchCallback watch) { - TestKeeperListRequest request; + TestKeeperFilteredListRequest request; request.path = path; + request.list_request_type = list_request_type; RequestInfo request_info; request_info.request = std::make_shared(std::move(request)); diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index 40cac3094f1..6e77a5d38c1 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -71,6 +71,7 @@ public: void list( const String & path, + ListRequestType list_request_type, ListCallback callback, WatchCallback watch) override; diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index c8ae6b72c3e..5a0be0f76ff 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "Common/ZooKeeper/IKeeper.h" #include #include #include @@ -312,9 +313,10 @@ static Coordination::WatchCallback callbackForEvent(const EventPtr & watch) Coordination::Error ZooKeeper::getChildrenImpl(const std::string & path, Strings & res, Coordination::Stat * stat, - Coordination::WatchCallback watch_callback) + Coordination::WatchCallback watch_callback, + Coordination::ListRequestType list_request_type) { - auto future_result = asyncTryGetChildrenNoThrow(path, watch_callback); + auto future_result = asyncTryGetChildrenNoThrow(path, watch_callback, list_request_type); if (future_result.wait_for(std::chrono::milliseconds(operation_timeout_ms)) != std::future_status::ready) { @@ -335,26 +337,28 @@ Coordination::Error ZooKeeper::getChildrenImpl(const std::string & path, Strings } } -Strings ZooKeeper::getChildren( - const std::string & path, Coordination::Stat * stat, const EventPtr & watch) +Strings ZooKeeper::getChildren(const std::string & path, Coordination::Stat * stat, const EventPtr & watch) { Strings res; check(tryGetChildren(path, res, stat, watch), path); return res; } -Strings ZooKeeper::getChildrenWatch( - const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) +Strings ZooKeeper::getChildrenWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) { Strings res; check(tryGetChildrenWatch(path, res, stat, watch_callback), path); return res; } -Coordination::Error ZooKeeper::tryGetChildren(const std::string & path, Strings & res, - Coordination::Stat * stat, const EventPtr & watch) +Coordination::Error ZooKeeper::tryGetChildren( + const std::string & path, + Strings & res, + Coordination::Stat * stat, + const EventPtr & watch, + Coordination::ListRequestType list_request_type) { - Coordination::Error code = getChildrenImpl(path, res, stat, callbackForEvent(watch)); + Coordination::Error code = getChildrenImpl(path, res, stat, callbackForEvent(watch), list_request_type); if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE)) throw KeeperException(code, path); @@ -362,10 +366,14 @@ Coordination::Error ZooKeeper::tryGetChildren(const std::string & path, Strings return code; } -Coordination::Error ZooKeeper::tryGetChildrenWatch(const std::string & path, Strings & res, - Coordination::Stat * stat, Coordination::WatchCallback watch_callback) +Coordination::Error ZooKeeper::tryGetChildrenWatch( + const std::string & path, + Strings & res, + Coordination::Stat * stat, + Coordination::WatchCallback watch_callback, + Coordination::ListRequestType list_request_type) { - Coordination::Error code = getChildrenImpl(path, res, stat, watch_callback); + Coordination::Error code = getChildrenImpl(path, res, stat, watch_callback, list_request_type); if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE)) throw KeeperException(code, path); @@ -1046,7 +1054,8 @@ std::future ZooKeeper::asyncTrySetNoThrow(const std:: return future; } -std::future ZooKeeper::asyncGetChildren(const std::string & path, Coordination::WatchCallback watch_callback) +std::future ZooKeeper::asyncGetChildren( + const std::string & path, Coordination::WatchCallback watch_callback, Coordination::ListRequestType list_request_type) { auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -1059,11 +1068,12 @@ std::future ZooKeeper::asyncGetChildren(const std::s promise->set_value(response); }; - impl->list(path, std::move(callback), watch_callback); + impl->list(path, list_request_type, std::move(callback), watch_callback); return future; } -std::future ZooKeeper::asyncTryGetChildrenNoThrow(const std::string & path, Coordination::WatchCallback watch_callback) +std::future ZooKeeper::asyncTryGetChildrenNoThrow( + const std::string & path, Coordination::WatchCallback watch_callback, Coordination::ListRequestType list_request_type) { auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -1073,7 +1083,7 @@ std::future ZooKeeper::asyncTryGetChildrenNoThrow(co promise->set_value(response); }; - impl->list(path, std::move(callback), watch_callback); + impl->list(path, list_request_type, std::move(callback), watch_callback); return future; } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 6aebccd2b4e..d2f92b6b4c3 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -194,11 +194,13 @@ public: /// * The node doesn't exist. Coordination::Error tryGetChildren(const std::string & path, Strings & res, Coordination::Stat * stat = nullptr, - const EventPtr & watch = nullptr); + const EventPtr & watch = nullptr, + Coordination::ListRequestType list_request_type = Coordination::ListRequestType::ALL); Coordination::Error tryGetChildrenWatch(const std::string & path, Strings & res, Coordination::Stat * stat, - Coordination::WatchCallback watch_callback); + Coordination::WatchCallback watch_callback, + Coordination::ListRequestType list_request_type = Coordination::ListRequestType::ALL); /// Performs several operations in a transaction. /// Throws on every error. @@ -279,9 +281,15 @@ public: FutureExists asyncTryExistsNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {}); using FutureGetChildren = std::future; - FutureGetChildren asyncGetChildren(const std::string & path, Coordination::WatchCallback watch_callback = {}); + FutureGetChildren asyncGetChildren( + const std::string & path, + Coordination::WatchCallback watch_callback = {}, + Coordination::ListRequestType list_request_type = Coordination::ListRequestType::ALL); /// Like the previous one but don't throw any exceptions on future.get() - FutureGetChildren asyncTryGetChildrenNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {}); + FutureGetChildren asyncTryGetChildrenNoThrow( + const std::string & path, + Coordination::WatchCallback watch_callback = {}, + Coordination::ListRequestType list_request_type = Coordination::ListRequestType::ALL); using FutureSet = std::future; FutureSet asyncSet(const std::string & path, const std::string & data, int32_t version = -1); @@ -335,7 +343,11 @@ private: const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); Coordination::Error setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat); Coordination::Error getChildrenImpl( - const std::string & path, Strings & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback); + const std::string & path, + Strings & res, + Coordination::Stat * stat, + Coordination::WatchCallback watch_callback, + Coordination::ListRequestType list_request_type); Coordination::Error multiImpl(const Coordination::Requests & requests, Coordination::Responses & responses); Coordination::Error existsImpl(const std::string & path, Coordination::Stat * stat_, Coordination::WatchCallback watch_callback); Coordination::Error syncImpl(const std::string & path, std::string & returned_path); diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index de2fb630848..837ea5bbad8 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -1,3 +1,4 @@ +#include "Common/ZooKeeper/IKeeper.h" #include #include #include @@ -298,6 +299,32 @@ std::string ZooKeeperListRequest::toStringImpl() const return fmt::format("path = {}", path); } +void ZooKeeperFilteredListRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(has_watch, out); + Coordination::write(static_cast(list_request_type), out); +} + +void ZooKeeperFilteredListRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(has_watch, in); + + uint8_t read_request_type{0}; + Coordination::read(read_request_type, in); + list_request_type = static_cast(read_request_type); +} + +std::string ZooKeeperFilteredListRequest::toStringImpl() const +{ + return fmt::format( + "path = {}\n" + "list_request_type = {}", + path, + list_request_type); +} + void ZooKeeperListResponse::readImpl(ReadBuffer & in) { Coordination::read(names, in); diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index c7bfbe95b74..09f797fb47b 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -347,6 +347,18 @@ struct ZooKeeperSimpleListRequest final : ZooKeeperListRequest OpNum getOpNum() const override { return OpNum::SimpleList; } }; +struct ZooKeeperFilteredListRequest final : ZooKeeperListRequest +{ + ListRequestType list_request_type{ListRequestType::ALL}; + + OpNum getOpNum() const override { return OpNum::FilteredList; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + std::string toStringImpl() const override; + + size_t bytesSize() const override { return ZooKeeperListRequest::bytesSize() + sizeof(list_request_type); } +}; + struct ZooKeeperListResponse : ListResponse, ZooKeeperResponse { void readImpl(ReadBuffer & in) override; diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.cpp b/src/Common/ZooKeeper/ZooKeeperConstants.cpp index b0a05fe6c8d..5b121ed6138 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.cpp +++ b/src/Common/ZooKeeper/ZooKeeperConstants.cpp @@ -64,6 +64,8 @@ std::string toString(OpNum op_num) return "SetACL"; case OpNum::GetACL: return "GetACL"; + case OpNum::FilteredList: + return "FilteredList"; } int32_t raw_op = static_cast(op_num); throw Exception("Operation " + std::to_string(raw_op) + " is unknown", Error::ZUNIMPLEMENTED); diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 1ed2c442f6c..44f8437f12c 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -32,6 +32,10 @@ enum class OpNum : int32_t Check = 13, Multi = 14, Auth = 100, + + // CH Keeper specific operations + FilteredList = 500, + SessionID = 997, /// Special internal request }; diff --git a/src/Common/ZooKeeper/ZooKeeperIO.cpp b/src/Common/ZooKeeper/ZooKeeperIO.cpp index 066aa1a24f6..c84a8624d78 100644 --- a/src/Common/ZooKeeper/ZooKeeperIO.cpp +++ b/src/Common/ZooKeeper/ZooKeeperIO.cpp @@ -28,6 +28,11 @@ void write(int32_t x, WriteBuffer & out) writeBinary(x, out); } +void write(uint8_t x, WriteBuffer & out) +{ + writeBinary(x, out); +} + void write(OpNum x, WriteBuffer & out) { write(static_cast(x), out); @@ -91,6 +96,11 @@ void read(int64_t & x, ReadBuffer & in) x = __builtin_bswap64(x); } +void read(uint8_t & x, ReadBuffer & in) +{ + readBinary(x, in); +} + void read(int32_t & x, ReadBuffer & in) { readBinary(x, in); diff --git a/src/Common/ZooKeeper/ZooKeeperIO.h b/src/Common/ZooKeeper/ZooKeeperIO.h index c2c6149cd11..ec77b46f3d9 100644 --- a/src/Common/ZooKeeper/ZooKeeperIO.h +++ b/src/Common/ZooKeeper/ZooKeeperIO.h @@ -22,6 +22,7 @@ void write(uint64_t x, WriteBuffer & out); void write(int64_t x, WriteBuffer & out); void write(int32_t x, WriteBuffer & out); +void write(uint8_t x, WriteBuffer & out); void write(OpNum x, WriteBuffer & out); void write(bool x, WriteBuffer & out); void write(const std::string & s, WriteBuffer & out); @@ -50,6 +51,7 @@ void read(uint64_t & x, ReadBuffer & in); #endif void read(int64_t & x, ReadBuffer & in); void read(int32_t & x, ReadBuffer & in); +void read(uint8_t & x, ReadBuffer & in); void read(OpNum & x, ReadBuffer & in); void read(bool & x, ReadBuffer & in); void read(int8_t & x, ReadBuffer & in); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index bd284ed0c91..8fa6f28c29c 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1168,11 +1168,13 @@ void ZooKeeper::set( void ZooKeeper::list( const String & path, + ListRequestType list_request_type, ListCallback callback, WatchCallback watch) { - ZooKeeperListRequest request; + ZooKeeperFilteredListRequest request; request.path = path; + request.list_request_type = list_request_type; RequestInfo request_info; request_info.request = std::make_shared(std::move(request)); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index c4acaf8d1ee..aa27b0eefe9 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -164,6 +164,7 @@ public: void list( const String & path, + ListRequestType list_request_type, ListCallback callback, WatchCallback watch) override; diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 368b23f34d2..1b399e8cc92 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -289,6 +289,9 @@ void KeeperStateMachine::rollback(uint64_t log_idx, nuraft::buffer & data) if (!request_for_session.zxid) request_for_session.zxid = log_idx; + if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID) + return; + std::lock_guard lock(storage_and_responses_lock); storage->rollbackRequest(request_for_session.zxid); } diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 21265f0bd61..fd1fab5b6b0 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -6,6 +6,7 @@ #include #include #include +#include "Common/ZooKeeper/ZooKeeperCommon.h" #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace DB { @@ -1161,6 +1163,7 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc } auto & container = storage.container; + auto node_it = container.find(request.path); if (node_it == container.end()) { @@ -1178,8 +1181,31 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc const auto & children = node_it->value.getChildren(); response.names.reserve(children.size()); + const auto add_child = [&](const auto child) + { + using enum Coordination::ListRequestType; + + auto list_request_type = ALL; + if (auto * filtered_list = dynamic_cast(&request)) + list_request_type = filtered_list->list_request_type; + + if (list_request_type == ALL) + return true; + + auto child_path = (std::filesystem::path(request.path) / child.toView()).generic_string(); + auto child_it = container.find(child_path); + if (child_it == container.end()) + onStorageInconsistency(); + + const auto is_ephemeral = child_it->value.stat.ephemeralOwner != 0; + return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY); + }; + for (const auto child : children) - response.names.push_back(child.toString()); + { + if (add_child(child)) + response.names.push_back(child.toString()); + } response.stat = node_it->value.stat; response.error = Coordination::Error::ZOK; @@ -1623,7 +1649,7 @@ struct KeeperStorageAuthRequestProcessor final : public KeeperStorageRequestProc void KeeperStorage::finalize() { if (finalized) - throw DB::Exception("Testkeeper storage already finalized", ErrorCodes::LOGICAL_ERROR); + throw DB::Exception("KeeperStorage already finalized", ErrorCodes::LOGICAL_ERROR); finalized = true; @@ -1689,6 +1715,7 @@ KeeperStorageRequestProcessorsFactory::KeeperStorageRequestProcessorsFactory() registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); + registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); registerKeeperRequestProcessor(*this); diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index ee75f2a0860..bd0d329ef8d 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -1956,6 +1957,84 @@ TEST_P(CoordinationTest, TestUncommittedStateBasicCrud) ASSERT_FALSE(get_committed_data()); } +TEST_P(CoordinationTest, TestListRequestTypes) +{ + using namespace DB; + using namespace Coordination; + + KeeperStorage storage{500, "", true}; + + int64_t zxid = 0; + + static constexpr std::string_view path = "/test"; + + const auto create_path = [&](bool is_ephemeral) + { + const auto create_request = std::make_shared(); + int new_zxid = ++zxid; + create_request->path = path; + create_request->is_sequential = true; + create_request->is_ephemeral = is_ephemeral; + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_GE(responses.size(), 1); + const auto & create_response = dynamic_cast(*responses[0].response); + return create_response.path_created; + }; + + static constexpr size_t persistent_num = 5; + std::unordered_set expected_persistent_children; + for (size_t i = 0; i < persistent_num; ++i) + { + expected_persistent_children.insert(getBaseName(create_path(false)).toString()); + } + ASSERT_EQ(expected_persistent_children.size(), persistent_num); + + static constexpr size_t ephemeral_num = 5; + std::unordered_set expected_ephemeral_children; + for (size_t i = 0; i < ephemeral_num; ++i) + { + expected_ephemeral_children.insert(getBaseName(create_path(true)).toString()); + } + ASSERT_EQ(expected_ephemeral_children.size(), ephemeral_num); + + const auto get_children = [&](const auto list_request_type) + { + const auto list_request = std::make_shared(); + int new_zxid = ++zxid; + list_request->path = parentPath(StringRef{path}).toString(); + list_request->list_request_type = list_request_type; + storage.preprocessRequest(list_request, 1, 0, new_zxid); + auto responses = storage.processRequest(list_request, 1, new_zxid); + + EXPECT_GE(responses.size(), 1); + const auto & list_response = dynamic_cast(*responses[0].response); + return list_response.names; + }; + + const auto persistent_children = get_children(ListRequestType::PERSISTENT_ONLY); + EXPECT_EQ(persistent_children.size(), persistent_num); + for (const auto & child : persistent_children) + { + EXPECT_TRUE(expected_persistent_children.contains(child)) << "Missing persistent child " << child; + } + + const auto ephemeral_children = get_children(ListRequestType::EPHEMERAL_ONLY); + EXPECT_EQ(ephemeral_children.size(), ephemeral_num); + for (const auto & child : ephemeral_children) + { + EXPECT_TRUE(expected_ephemeral_children.contains(child)) << "Missing ephemeral child " << child; + } + + const auto all_children = get_children(ListRequestType::ALL); + EXPECT_EQ(all_children.size(), ephemeral_num + persistent_num); + for (const auto & child : all_children) + { + EXPECT_TRUE(expected_ephemeral_children.contains(child) || expected_persistent_children.contains(child)) << "Missing child " << child; + } +} + INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, CoordinationTest, ::testing::ValuesIn(std::initializer_list{ diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 2f37d2ea951..3a4b66e6266 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -559,4 +559,31 @@ String toString(const Field & x) x); } +String fieldTypeToString(Field::Types::Which type) +{ + switch (type) + { + case Field::Types::Which::Null: return "Null"; + case Field::Types::Which::Array: return "Array"; + case Field::Types::Which::Tuple: return "Tuple"; + case Field::Types::Which::Map: return "Map"; + case Field::Types::Which::Object: return "Object"; + case Field::Types::Which::AggregateFunctionState: return "AggregateFunctionState"; + case Field::Types::Which::Bool: return "Bool"; + case Field::Types::Which::String: return "String"; + case Field::Types::Which::Decimal32: return "Decimal32"; + case Field::Types::Which::Decimal64: return "Decimal64"; + case Field::Types::Which::Decimal128: return "Decimal128"; + case Field::Types::Which::Decimal256: return "Decimal256"; + case Field::Types::Which::Float64: return "Float64"; + case Field::Types::Which::Int64: return "Int64"; + case Field::Types::Which::Int128: return "Int128"; + case Field::Types::Which::Int256: return "Int256"; + case Field::Types::Which::UInt64: return "UInt64"; + case Field::Types::Which::UInt128: return "UInt128"; + case Field::Types::Which::UInt256: return "UInt256"; + case Field::Types::Which::UUID: return "UUID"; + } +} + } diff --git a/src/Core/Field.h b/src/Core/Field.h index 4948ec4ae61..08274876914 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -1011,6 +1011,8 @@ void writeFieldText(const Field & x, WriteBuffer & buf); String toString(const Field & x); +String fieldTypeToString(Field::Types::Which type); + } template <> diff --git a/src/Core/PostgreSQL/Connection.cpp b/src/Core/PostgreSQL/Connection.cpp index dfdf14d3506..5a589a80d02 100644 --- a/src/Core/PostgreSQL/Connection.cpp +++ b/src/Core/PostgreSQL/Connection.cpp @@ -73,6 +73,7 @@ void Connection::connect() if (!connection || !connection->is_open()) updateConnection(); } + } #endif diff --git a/src/Core/PostgreSQL/Connection.h b/src/Core/PostgreSQL/Connection.h index 97ce3c152d5..d39659a9953 100644 --- a/src/Core/PostgreSQL/Connection.h +++ b/src/Core/PostgreSQL/Connection.h @@ -32,7 +32,10 @@ struct ConnectionInfo class Connection : private boost::noncopyable { public: - explicit Connection(const ConnectionInfo & connection_info_, bool replication_ = false, size_t num_tries = 3); + explicit Connection( + const ConnectionInfo & connection_info_, + bool replication_ = false, + size_t num_tries = 3); void execWithRetry(const std::function & exec); diff --git a/src/Core/PostgreSQL/ConnectionHolder.h b/src/Core/PostgreSQL/ConnectionHolder.h index 38e321e222c..2fd8717c643 100644 --- a/src/Core/PostgreSQL/ConnectionHolder.h +++ b/src/Core/PostgreSQL/ConnectionHolder.h @@ -20,11 +20,20 @@ class ConnectionHolder { public: - ConnectionHolder(PoolPtr pool_, ConnectionPtr connection_) : pool(pool_), connection(std::move(connection_)) {} + ConnectionHolder(PoolPtr pool_, ConnectionPtr connection_, bool auto_close_) + : pool(pool_) + , connection(std::move(connection_)) + , auto_close(auto_close_) + {} ConnectionHolder(const ConnectionHolder & other) = delete; - ~ConnectionHolder() { pool->returnObject(std::move(connection)); } + ~ConnectionHolder() + { + if (auto_close) + connection.reset(); + pool->returnObject(std::move(connection)); + } pqxx::connection & get() { @@ -39,6 +48,7 @@ public: private: PoolPtr pool; ConnectionPtr connection; + bool auto_close; }; using ConnectionHolderPtr = std::unique_ptr; diff --git a/src/Core/PostgreSQL/PoolWithFailover.cpp b/src/Core/PostgreSQL/PoolWithFailover.cpp index 844c60087e0..1bac17de579 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.cpp +++ b/src/Core/PostgreSQL/PoolWithFailover.cpp @@ -5,6 +5,7 @@ #include "Utils.h" #include #include +#include #include #include @@ -20,10 +21,14 @@ namespace postgres { PoolWithFailover::PoolWithFailover( - const DB::ExternalDataSourcesConfigurationByPriority & configurations_by_priority, - size_t pool_size, size_t pool_wait_timeout_, size_t max_tries_) - : pool_wait_timeout(pool_wait_timeout_) - , max_tries(max_tries_) + const DB::ExternalDataSourcesConfigurationByPriority & configurations_by_priority, + size_t pool_size, + size_t pool_wait_timeout_, + size_t max_tries_, + bool auto_close_connection_) + : pool_wait_timeout(pool_wait_timeout_) + , max_tries(max_tries_) + , auto_close_connection(auto_close_connection_) { LOG_TRACE(&Poco::Logger::get("PostgreSQLConnectionPool"), "PostgreSQL connection pool size: {}, connection wait timeout: {}, max failover tries: {}", pool_size, pool_wait_timeout, max_tries_); @@ -40,10 +45,14 @@ PoolWithFailover::PoolWithFailover( } PoolWithFailover::PoolWithFailover( - const DB::StoragePostgreSQLConfiguration & configuration, - size_t pool_size, size_t pool_wait_timeout_, size_t max_tries_) + const DB::StoragePostgreSQLConfiguration & configuration, + size_t pool_size, + size_t pool_wait_timeout_, + size_t max_tries_, + bool auto_close_connection_) : pool_wait_timeout(pool_wait_timeout_) , max_tries(max_tries_) + , auto_close_connection(auto_close_connection_) { LOG_TRACE(&Poco::Logger::get("PostgreSQLConnectionPool"), "PostgreSQL connection pool size: {}, connection wait timeout: {}, max failover tries: {}", pool_size, pool_wait_timeout, max_tries_); @@ -94,7 +103,9 @@ ConnectionHolderPtr PoolWithFailover::get() catch (const pqxx::broken_connection & pqxx_error) { LOG_ERROR(log, "Connection error: {}", pqxx_error.what()); - error_message << "Try " << try_idx + 1 << ". Connection to `" << replica.connection_info.host_port << "` failed: " << pqxx_error.what() << "\n"; + error_message << fmt::format( + "Try {}. Connection to {} failed with error: {}\n", + try_idx + 1, DB::backQuote(replica.connection_info.host_port), pqxx_error.what()); replica.pool->returnObject(std::move(connection)); continue; @@ -105,7 +116,7 @@ ConnectionHolderPtr PoolWithFailover::get() throw; } - auto connection_holder = std::make_unique(replica.pool, std::move(connection)); + auto connection_holder = std::make_unique(replica.pool, std::move(connection), auto_close_connection); /// Move all traversed replicas to the end. if (replicas.size() > 1) diff --git a/src/Core/PostgreSQL/PoolWithFailover.h b/src/Core/PostgreSQL/PoolWithFailover.h index 4e3a17b5e9c..81c94d92141 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.h +++ b/src/Core/PostgreSQL/PoolWithFailover.h @@ -12,6 +12,10 @@ #include +static constexpr inline auto POSTGRESQL_POOL_DEFAULT_SIZE = 16; +static constexpr inline auto POSTGRESQL_POOL_WAIT_TIMEOUT = 5000; +static constexpr inline auto POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES = 5; + namespace postgres { @@ -21,21 +25,19 @@ class PoolWithFailover using RemoteDescription = std::vector>; public: - static constexpr inline auto POSTGRESQL_POOL_DEFAULT_SIZE = 16; - static constexpr inline auto POSTGRESQL_POOL_WAIT_TIMEOUT = 5000; - static constexpr inline auto POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES = 5; - - explicit PoolWithFailover( + PoolWithFailover( const DB::ExternalDataSourcesConfigurationByPriority & configurations_by_priority, - size_t pool_size = POSTGRESQL_POOL_DEFAULT_SIZE, - size_t pool_wait_timeout = POSTGRESQL_POOL_WAIT_TIMEOUT, - size_t max_tries_ = POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + size_t pool_size, + size_t pool_wait_timeout, + size_t max_tries_, + bool auto_close_connection_); explicit PoolWithFailover( const DB::StoragePostgreSQLConfiguration & configuration, - size_t pool_size = POSTGRESQL_POOL_DEFAULT_SIZE, - size_t pool_wait_timeout = POSTGRESQL_POOL_WAIT_TIMEOUT, - size_t max_tries_ = POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + size_t pool_size, + size_t pool_wait_timeout, + size_t max_tries_, + bool auto_close_connection_); PoolWithFailover(const PoolWithFailover & other) = delete; @@ -58,6 +60,7 @@ private: ReplicasWithPriority replicas_with_priority; size_t pool_wait_timeout; size_t max_tries; + bool auto_close_connection; std::mutex mutex; Poco::Logger * log = &Poco::Logger::get("PostgreSQLConnectionPool"); }; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 03c0613075e..03f3ccf1bce 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -96,7 +96,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited. Only has meaning at server startup.", 0) \ M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited. Only has meaning at server startup.", 0) \ - M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ and FileLog engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \ + M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \ + M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \ \ M(Milliseconds, distributed_directory_monitor_sleep_time_ms, 100, "Sleep time for StorageDistributed DirectoryMonitors, in case of any errors delay grows exponentially.", 0) \ M(Milliseconds, distributed_directory_monitor_max_sleep_time_ms, 30000, "Maximum sleep time for StorageDistributed DirectoryMonitors, it limits exponential growth too.", 0) \ @@ -428,6 +429,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ + M(Bool, postgresql_connection_pool_auto_close_connection, false, "Close connection before returning connection to the pool.", 0) \ M(UInt64, glob_expansion_max_elements, 1000, "Maximum number of allowed addresses (For external storages, table functions, etc).", 0) \ M(UInt64, odbc_bridge_connection_pool_size, 16, "Connection pool size for each connection settings string in ODBC bridge.", 0) \ M(Bool, odbc_bridge_use_connection_pooling, true, "Use connection pooling in ODBC bridge. If set to false, a new connection is created every time", 0) \ @@ -763,6 +765,12 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(String, input_format_mysql_dump_table_name, "", "Name of the table in MySQL dump from which to read data", 0) \ M(Bool, input_format_mysql_dump_map_column_names, true, "Match columns from table in MySQL dump and columns from ClickHouse table by names", 0) \ + \ + M(UInt64, output_format_sql_insert_max_batch_size, DEFAULT_BLOCK_SIZE, "The maximum number of rows in one INSERT statement.", 0) \ + M(String, output_format_sql_insert_table_name, "table", "The name of table in the output INSERT query", 0) \ + M(Bool, output_format_sql_insert_include_column_names, true, "Include column names in INSERT query", 0) \ + M(Bool, output_format_sql_insert_use_replace, false, "Use REPLACE statement instead of INSERT", 0) \ + M(Bool, output_format_sql_insert_quote_names, true, "Quote column names with '`' characters", 0) \ // End of FORMAT_FACTORY_SETTINGS // Please add settings non-related to formats into the COMMON_SETTINGS above. diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index bff1971bad9..66d38337243 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -30,12 +30,13 @@ IMPLEMENT_SETTING_ENUM(JoinStrictness, ErrorCodes::UNKNOWN_JOIN, {"ANY", JoinStrictness::ANY}}) -IMPLEMENT_SETTING_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN, +IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN, {{"auto", JoinAlgorithm::AUTO}, {"hash", JoinAlgorithm::HASH}, {"partial_merge", JoinAlgorithm::PARTIAL_MERGE}, {"prefer_partial_merge", JoinAlgorithm::PREFER_PARTIAL_MERGE}, - {"parallel_hash", JoinAlgorithm::PARALLEL_HASH}}) + {"parallel_hash", JoinAlgorithm::PARALLEL_HASH}, + {"direct", JoinAlgorithm::DIRECT}}) IMPLEMENT_SETTING_ENUM(TotalsMode, ErrorCodes::UNKNOWN_TOTALS_MODE, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 83a65f2a320..bb9311a808b 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -43,9 +43,10 @@ enum class JoinAlgorithm PARTIAL_MERGE, PREFER_PARTIAL_MERGE, PARALLEL_HASH, + DIRECT, }; -DECLARE_SETTING_ENUM(JoinAlgorithm) +DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm) /// Which rows should be included in TOTALS. diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 3fc2503aaa5..46c77593d4e 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -6,6 +6,7 @@ #cmakedefine01 USE_MYSQL #cmakedefine01 USE_RDKAFKA #cmakedefine01 USE_AMQPCPP +#cmakedefine01 USE_NATSIO #cmakedefine01 USE_EMBEDDED_COMPILER #cmakedefine01 USE_SSL #cmakedefine01 USE_LDAP diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index af82d382063..df1e58ca852 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -344,9 +344,13 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String use_table_cache = safeGetLiteralValue(engine_args[5], engine_name); } - auto pool = std::make_shared(configuration, - context->getSettingsRef().postgresql_connection_pool_size, - context->getSettingsRef().postgresql_connection_pool_wait_timeout); + const auto & settings = context->getSettingsRef(); + auto pool = std::make_shared( + configuration, + settings.postgresql_connection_pool_size, + settings.postgresql_connection_pool_wait_timeout, + POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + settings.postgresql_connection_pool_auto_close_connection); return std::make_shared( context, metadata_path, engine_define, database_name, configuration, pool, use_table_cache); diff --git a/src/Dictionaries/CMakeLists.txt b/src/Dictionaries/CMakeLists.txt index 19e82c45cc2..964e44efffe 100644 --- a/src/Dictionaries/CMakeLists.txt +++ b/src/Dictionaries/CMakeLists.txt @@ -4,8 +4,7 @@ add_headers_and_sources(clickhouse_dictionaries .) add_headers_and_sources(clickhouse_dictionaries "${CMAKE_CURRENT_BINARY_DIR}/generated/") -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") - +if (OMIT_HEAVY_DEBUG_SYMBOLS) # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. set_source_files_properties( FlatDictionary.cpp @@ -15,7 +14,7 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELW RangeHashedDictionary.cpp DirectDictionary.cpp PROPERTIES COMPILE_FLAGS -g0) -endif () +endif() list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp DictionarySourceFactory.cpp DictionaryStructure.cpp getDictionaryConfigurationFromAST.cpp) list(REMOVE_ITEM clickhouse_dictionaries_headers DictionaryFactory.h DictionarySourceFactory.h DictionaryStructure.h getDictionaryConfigurationFromAST.h) diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index eb1a4caf2fc..42884278e7d 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -191,10 +191,13 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) const auto settings_config_prefix = config_prefix + ".postgresql"; auto has_config_key = [](const String & key) { return dictionary_allowed_keys.contains(key) || key.starts_with("replica"); }; auto configuration = getExternalDataSourceConfigurationByPriority(config, settings_config_prefix, context, has_config_key); + const auto & settings = context->getSettingsRef(); auto pool = std::make_shared( - configuration.replicas_configurations, - context->getSettingsRef().postgresql_connection_pool_size, - context->getSettingsRef().postgresql_connection_pool_wait_timeout); + configuration.replicas_configurations, + settings.postgresql_connection_pool_size, + settings.postgresql_connection_pool_wait_timeout, + POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + settings.postgresql_connection_pool_auto_close_connection); PostgreSQLDictionarySource::Configuration dictionary_configuration { diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index dc6344137d2..756b33d3eb2 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -158,6 +158,11 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.column_names_for_schema_inference = settings.column_names_for_schema_inference; format_settings.mysql_dump.table_name = settings.input_format_mysql_dump_table_name; format_settings.mysql_dump.map_column_names = settings.input_format_mysql_dump_map_column_names; + format_settings.sql_insert.max_batch_size = settings.output_format_sql_insert_max_batch_size; + format_settings.sql_insert.include_column_names = settings.output_format_sql_insert_include_column_names; + format_settings.sql_insert.table_name = settings.output_format_sql_insert_table_name; + format_settings.sql_insert.use_replace = settings.output_format_sql_insert_use_replace; + format_settings.sql_insert.quote_names = settings.output_format_sql_insert_quote_names; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context if (format_settings.schema.is_server) diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 0032aea57e4..70bf8979383 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -274,6 +275,15 @@ struct FormatSettings String table_name; bool map_column_names = true; } mysql_dump; + + struct + { + UInt64 max_batch_size = DEFAULT_BLOCK_SIZE; + String table_name = "table"; + bool include_column_names = true; + bool use_replace = false; + bool quote_names = true; + } sql_insert; }; } diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp index 8493c84173d..0953572fab9 100644 --- a/src/Formats/registerFormats.cpp +++ b/src/Formats/registerFormats.cpp @@ -82,6 +82,7 @@ void registerOutputFormatMySQLWire(FormatFactory & factory); void registerOutputFormatMarkdown(FormatFactory & factory); void registerOutputFormatPostgreSQLWire(FormatFactory & factory); void registerOutputFormatPrometheus(FormatFactory & factory); +void registerOutputFormatSQLInsert(FormatFactory & factory); /// Input only formats. @@ -205,6 +206,7 @@ void registerFormats() registerOutputFormatPostgreSQLWire(factory); registerOutputFormatCapnProto(factory); registerOutputFormatPrometheus(factory); + registerOutputFormatSQLInsert(factory); registerInputFormatRegexp(factory); registerInputFormatJSONAsString(factory); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 1f728489b73..7d7aba6955f 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -35,25 +35,8 @@ if (TARGET OpenSSL::Crypto) target_link_libraries(clickhouse_functions PUBLIC OpenSSL::Crypto) endif() -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" - OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" - OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") - set (STRIP_DSF_DEFAULT ON) -else() - set (STRIP_DSF_DEFAULT OFF) -endif() - - -# Provides faster linking and lower binary size. -# Tradeoff is the inability to debug some source files with e.g. gdb -# (empty stack frames and no local variables)." -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS "Do not generate debugger info for ClickHouse functions" ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(INFO "Not generating debugger info for ClickHouse functions") +if (OMIT_HEAVY_DEBUG_SYMBOLS) target_compile_options(clickhouse_functions PRIVATE "-g0") -else() - message(STATUS "Generating debugger info for ClickHouse functions") endif() if (TARGET ch_contrib::icu) diff --git a/src/Functions/GatherUtils/CMakeLists.txt b/src/Functions/GatherUtils/CMakeLists.txt index 460b02326a1..fe600f86d07 100644 --- a/src/Functions/GatherUtils/CMakeLists.txt +++ b/src/Functions/GatherUtils/CMakeLists.txt @@ -12,7 +12,7 @@ if (HAS_SUGGEST_DESTRUCTOR_OVERRIDE) target_compile_definitions(clickhouse_functions_gatherutils PUBLIC HAS_SUGGEST_DESTRUCTOR_OVERRIDE) endif() -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) +if (OMIT_HEAVY_DEBUG_SYMBOLS) target_compile_options(clickhouse_functions_gatherutils PRIVATE "-g0") endif() diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index 2531a0f84e6..bdc826008eb 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -8,6 +8,6 @@ target_link_libraries(clickhouse_functions_jsonpath PRIVATE dbms) target_link_libraries(clickhouse_functions_jsonpath PRIVATE clickhouse_parsers) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath) -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) +if (OMIT_HEAVY_DEBUG_SYMBOLS) target_compile_options(clickhouse_functions_jsonpath PRIVATE "-g0") endif() diff --git a/src/Functions/URL/CMakeLists.txt b/src/Functions/URL/CMakeLists.txt index a1af30e6f81..28a72651315 100644 --- a/src/Functions/URL/CMakeLists.txt +++ b/src/Functions/URL/CMakeLists.txt @@ -3,7 +3,7 @@ add_headers_and_sources(clickhouse_functions_url .) add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers}) target_link_libraries(clickhouse_functions_url PRIVATE dbms) -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) +if (OMIT_HEAVY_DEBUG_SYMBOLS) target_compile_options(clickhouse_functions_url PRIVATE "-g0") endif() diff --git a/src/Functions/array/CMakeLists.txt b/src/Functions/array/CMakeLists.txt index 9762674d6e9..6b4a8122d16 100644 --- a/src/Functions/array/CMakeLists.txt +++ b/src/Functions/array/CMakeLists.txt @@ -3,6 +3,6 @@ add_headers_and_sources(clickhouse_functions_array .) add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils) -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) +if (OMIT_HEAVY_DEBUG_SYMBOLS) target_compile_options(clickhouse_functions_array PRIVATE "-g0") endif() diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h index 9881d463ed4..45b98f8c977 100644 --- a/src/IO/ParallelReadBuffer.h +++ b/src/IO/ParallelReadBuffer.h @@ -47,6 +47,7 @@ public: off_t getPosition() override; const ReadBufferFactory & getReadBufferFactory() const { return *reader_factory; } + ReadBufferFactory & getReadBufferFactory() { return *reader_factory; } private: /// Reader in progress with a list of read segments diff --git a/src/IO/WithFileSize.cpp b/src/IO/WithFileSize.cpp index 28542db7a73..f71690fcdee 100644 --- a/src/IO/WithFileSize.cpp +++ b/src/IO/WithFileSize.cpp @@ -33,6 +33,10 @@ size_t getFileSizeFromReadBuffer(ReadBuffer & in) { return getFileSize(compressed->getWrappedReadBuffer()); } + else if (auto * parallel = dynamic_cast(&in)) + { + return getFileSize(parallel->getReadBufferFactory()); + } return getFileSize(in); } @@ -47,6 +51,10 @@ bool isBufferWithFileSize(const ReadBuffer & in) { return isBufferWithFileSize(compressed->getWrappedReadBuffer()); } + else if (const auto * parallel = dynamic_cast(&in)) + { + return dynamic_cast(¶llel->getReadBufferFactory()) != nullptr; + } return dynamic_cast(&in) != nullptr; } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 511e5c9e031..a99ecee43bf 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -374,7 +374,15 @@ Block Aggregator::Params::getHeader( if (only_merge) { - res = header.cloneEmpty(); + NameSet needed_columns(keys.begin(), keys.end()); + for (const auto & aggregate : aggregates) + needed_columns.emplace(aggregate.column_name); + + for (const auto & column : header) + { + if (needed_columns.contains(column.name)) + res.insert(column.cloneEmpty()); + } if (final) { diff --git a/src/Interpreters/DirectJoin.cpp b/src/Interpreters/DirectJoin.cpp new file mode 100644 index 00000000000..21baeb31945 --- /dev/null +++ b/src/Interpreters/DirectJoin.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int UNSUPPORTED_JOIN_KEYS; + extern const int NOT_FOUND_COLUMN_IN_BLOCK; +} + +static Block originalRightBlock(const Block & block, const TableJoin & table_join) +{ + Block original_right_block; + for (const auto & col : block) + original_right_block.insert({col.column, col.type, table_join.getOriginalName(col.name)}); + return original_right_block; +} + +/// Converts `columns` from `source_sample_block` structure to `result_sample_block`. +/// Can select subset of columns and change types. +static MutableColumns convertBlockStructure( + const Block & source_sample_block, const Block & result_sample_block, MutableColumns && columns, const PaddedPODArray & null_map) +{ + MutableColumns result_columns; + for (const auto & out_sample_col : result_sample_block) + { + auto i = source_sample_block.getPositionByName(out_sample_col.name); + if (columns[i] == nullptr) + { + throw DB::Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, "Can't find column '{}'", out_sample_col.name); + } + + ColumnWithTypeAndName col = source_sample_block.getByPosition(i); + if (!col.type->equals(*out_sample_col.type)) + { + col.column = std::move(columns[i]); + result_columns.push_back(IColumn::mutate(castColumnAccurate(col, out_sample_col.type))); + } + else + { + result_columns.push_back(std::move(columns[i])); + } + columns[i] = nullptr; + + if (result_columns.back()->isNullable()) + { + assert_cast(result_columns.back().get())->applyNegatedNullMap(null_map); + } + } + return result_columns; +} + +DirectKeyValueJoin::DirectKeyValueJoin(std::shared_ptr table_join_, + const Block & right_sample_block_, + std::shared_ptr storage_) + : table_join(table_join_) + , storage(storage_) + , right_sample_block(right_sample_block_) + , log(&Poco::Logger::get("DirectKeyValueJoin")) +{ + if (!table_join->oneDisjunct() + || table_join->getOnlyClause().key_names_left.size() != 1 + || table_join->getOnlyClause().key_names_right.size() != 1) + { + throw DB::Exception(ErrorCodes::UNSUPPORTED_JOIN_KEYS, "Not supported by direct JOIN"); + } + + if (table_join->strictness() != ASTTableJoin::Strictness::All && + table_join->strictness() != ASTTableJoin::Strictness::Any && + table_join->strictness() != ASTTableJoin::Strictness::RightAny) + { + throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "Not supported by direct JOIN"); + } + + LOG_TRACE(log, "Using direct join"); +} + +bool DirectKeyValueJoin::addJoinedBlock(const Block &, bool) +{ + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Unreachable code reached"); +} + +void DirectKeyValueJoin::checkTypesOfKeys(const Block & block) const +{ + for (const auto & onexpr : table_join->getClauses()) + { + JoinCommon::checkTypesOfKeys(block, onexpr.key_names_left, right_sample_block, onexpr.key_names_right); + } +} + +void DirectKeyValueJoin::joinBlock(Block & block, std::shared_ptr &) +{ + const String & key_name = table_join->getOnlyClause().key_names_left[0]; + const ColumnWithTypeAndName & key_col = block.getByName(key_name); + if (!key_col.column) + return; + + NullMap null_map; + Chunk joined_chunk = storage->getByKeys({key_col}, null_map); + + /// Expected right block may differ from structure in storage, because of `join_use_nulls` or we just select not all columns. + Block original_right_block = originalRightBlock(right_sample_block, *table_join); + Block sample_storage_block = storage->getInMemoryMetadataPtr()->getSampleBlock(); + MutableColumns result_columns = convertBlockStructure(sample_storage_block, original_right_block, joined_chunk.mutateColumns(), null_map); + + for (size_t i = 0; i < result_columns.size(); ++i) + { + ColumnWithTypeAndName col = right_sample_block.getByPosition(i); + col.column = std::move(result_columns[i]); + block.insert(std::move(col)); + } + + if (!isLeftOrFull(table_join->kind())) + { + MutableColumns dst_columns = block.mutateColumns(); + for (auto & col : dst_columns) + { + col = IColumn::mutate(col->filter(null_map, -1)); + } + block.setColumns(std::move(dst_columns)); + } +} + +} diff --git a/src/Interpreters/DirectJoin.h b/src/Interpreters/DirectJoin.h new file mode 100644 index 00000000000..f7da06ef826 --- /dev/null +++ b/src/Interpreters/DirectJoin.h @@ -0,0 +1,60 @@ +#pragma once + +#include + +#include + +#include +#include + +#include + +#include +#include + +namespace DB +{ + +class NotJoinedBlocks; + +class DirectKeyValueJoin : public IJoin +{ +public: + DirectKeyValueJoin( + std::shared_ptr table_join_, + const Block & right_sample_block_, + std::shared_ptr storage_); + + virtual const TableJoin & getTableJoin() const override { return *table_join; } + + virtual bool addJoinedBlock(const Block &, bool) override; + virtual void checkTypesOfKeys(const Block &) const override; + + /// Join the block with data from left hand of JOIN to the right hand data (that was previously built by calls to addJoinedBlock). + /// Could be called from different threads in parallel. + virtual void joinBlock(Block & block, std::shared_ptr &) override; + + virtual size_t getTotalRowCount() const override { return 0; } + + virtual size_t getTotalByteCount() const override { return 0; } + + virtual bool alwaysReturnsEmptySet() const override { return false; } + + virtual bool isFilled() const override { return true; } + + virtual std::shared_ptr + getNonJoinedBlocks(const Block &, const Block &, UInt64) const override + { + return nullptr; + } + +private: + std::shared_ptr table_join; + std::shared_ptr storage; + Block right_sample_block; + Block sample_block_with_columns_to_add; + Poco::Logger * log; + +}; + +} diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 8d0c4dee023..66d852e31c1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -43,9 +44,12 @@ #include #include +#include #include #include #include +#include + #include #include @@ -81,6 +85,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int UNKNOWN_IDENTIFIER; extern const int UNKNOWN_TYPE_OF_AST_NODE; + extern const int UNSUPPORTED_METHOD; } namespace @@ -1066,24 +1071,24 @@ static ActionsDAGPtr createJoinedBlockActions(ContextPtr context, const TableJoi return ExpressionAnalyzer(expression_list, syntax_result, context).getActionsDAG(true, false); } -static std::shared_ptr chooseJoinAlgorithm(std::shared_ptr analyzed_join, const Block & sample_block, ContextPtr context) +static std::shared_ptr chooseJoinAlgorithm(std::shared_ptr analyzed_join, const Block & right_sample_block, ContextPtr context) { /// HashJoin with Dictionary optimisation - if (analyzed_join->tryInitDictJoin(sample_block, context)) - return std::make_shared(analyzed_join, sample_block); + if (analyzed_join->tryInitDictJoin(right_sample_block, context)) + return std::make_shared(analyzed_join, right_sample_block); bool allow_merge_join = analyzed_join->allowMergeJoin(); if (analyzed_join->forceHashJoin() || (analyzed_join->preferMergeJoin() && !allow_merge_join)) { if (analyzed_join->allowParallelHashJoin()) { - return std::make_shared(context, analyzed_join, context->getSettings().max_threads, sample_block); + return std::make_shared(context, analyzed_join, context->getSettings().max_threads, right_sample_block); } - return std::make_shared(analyzed_join, sample_block); + return std::make_shared(analyzed_join, right_sample_block); } else if (analyzed_join->forceMergeJoin() || (analyzed_join->preferMergeJoin() && allow_merge_join)) - return std::make_shared(analyzed_join, sample_block); - return std::make_shared(analyzed_join, sample_block); + return std::make_shared(analyzed_join, right_sample_block); + return std::make_shared(analyzed_join, right_sample_block); } static std::unique_ptr buildJoinedPlan( @@ -1094,12 +1099,12 @@ static std::unique_ptr buildJoinedPlan( { /// Actions which need to be calculated on joined block. auto joined_block_actions = createJoinedBlockActions(context, analyzed_join); - Names original_right_columns; - NamesWithAliases required_columns_with_aliases = analyzed_join.getRequiredColumns( Block(joined_block_actions->getResultColumns()), joined_block_actions->getRequiredColumns().getNames()); + + Names original_right_column_names; for (auto & pr : required_columns_with_aliases) - original_right_columns.push_back(pr.first); + original_right_column_names.push_back(pr.first); /** For GLOBAL JOINs (in the case, for example, of the push method for executing GLOBAL subqueries), the following occurs * - in the addExternalStorage function, the JOIN (SELECT ...) subquery is replaced with JOIN _data1, @@ -1110,18 +1115,18 @@ static std::unique_ptr buildJoinedPlan( auto interpreter = interpretSubquery( join_element.table_expression, context, - original_right_columns, + original_right_column_names, query_options.copy().setWithAllColumns().ignoreProjections(false).ignoreAlias(false)); auto joined_plan = std::make_unique(); interpreter->buildQueryPlan(*joined_plan); { - auto sample_block = interpreter->getSampleBlock(); - auto rename_dag = std::make_unique(sample_block.getColumnsWithTypeAndName()); + Block original_right_columns = interpreter->getSampleBlock(); + auto rename_dag = std::make_unique(original_right_columns.getColumnsWithTypeAndName()); for (const auto & name_with_alias : required_columns_with_aliases) { - if (sample_block.has(name_with_alias.first)) + if (name_with_alias.first != name_with_alias.second && original_right_columns.has(name_with_alias.first)) { - auto pos = sample_block.getPositionByName(name_with_alias.first); + auto pos = original_right_columns.getPositionByName(name_with_alias.first); const auto & alias = rename_dag->addAlias(*rename_dag->getInputs()[pos], name_with_alias.second); rename_dag->getIndex()[pos] = &alias; } @@ -1139,6 +1144,52 @@ static std::unique_ptr buildJoinedPlan( return joined_plan; } +std::shared_ptr tryKeyValueJoin(std::shared_ptr analyzed_join, const Block & right_sample_block) +{ + auto error_or_null = [&](const String & msg) + { + if (analyzed_join->isForcedAlgorithm(JoinAlgorithm::DIRECT)) + throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, "Can't use '{}' join algorithm: {}", JoinAlgorithm::DIRECT, msg); + return nullptr; + }; + + if (!analyzed_join->isAllowedAlgorithm(JoinAlgorithm::DIRECT)) + return nullptr; + + auto storage = analyzed_join->getStorageKeyValue(); + if (!storage) + return error_or_null("unsupported storage"); + + if (!isInnerOrLeft(analyzed_join->kind())) + return error_or_null("illegal kind"); + + if (analyzed_join->strictness() != ASTTableJoin::Strictness::All && + analyzed_join->strictness() != ASTTableJoin::Strictness::Any && + analyzed_join->strictness() != ASTTableJoin::Strictness::RightAny) + return error_or_null("illegal strictness"); + + const auto & clauses = analyzed_join->getClauses(); + bool only_one_key = clauses.size() == 1 && + clauses[0].key_names_left.size() == 1 && + clauses[0].key_names_right.size() == 1 && + !clauses[0].on_filter_condition_left && + !clauses[0].on_filter_condition_right; + + if (!only_one_key) + return error_or_null("multiple keys is not allowed"); + + String key_name = clauses[0].key_names_right[0]; + String original_key_name = analyzed_join->getOriginalName(key_name); + const auto & storage_primary_key = storage->getPrimaryKey(); + if (storage_primary_key.size() != 1 || storage_primary_key[0] != original_key_name) + { + return error_or_null(fmt::format("key '{}'{} doesn't match storage '{}'", + key_name, (key_name != original_key_name ? " (aka '" + original_key_name + "')" : ""), fmt::join(storage_primary_key, ","))); + } + + return std::make_shared(analyzed_join, right_sample_block, storage); +} + JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( const ASTTablesInSelectQueryElement & join_element, const ColumnsWithTypeAndName & left_columns, @@ -1171,7 +1222,14 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( joined_plan->addStep(std::move(converting_step)); } - JoinPtr join = chooseJoinAlgorithm(analyzed_join, joined_plan->getCurrentDataStream().header, getContext()); + const Block & right_sample_block = joined_plan->getCurrentDataStream().header; + if (JoinPtr kvjoin = tryKeyValueJoin(analyzed_join, right_sample_block)) + { + joined_plan.reset(); + return kvjoin; + } + + JoinPtr join = chooseJoinAlgorithm(analyzed_join, right_sample_block, getContext()); /// Do not make subquery for join over dictionary. if (analyzed_join->getDictionaryReader()) @@ -1539,7 +1597,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai if (optimize_read_in_order) { - for (auto & child : select_query->orderBy()->children) + for (const auto & child : select_query->orderBy()->children) { auto actions_dag = std::make_shared(columns_after_join); getRootActions(child, only_types, actions_dag); diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 0cec83e964b..a5bbcf9a373 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -22,9 +22,10 @@ #include #include - +#include #include +#include #include #include diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 17d09b25ea1..ae6b059696c 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,9 @@ #include +#include +#include + namespace DB { @@ -167,11 +171,6 @@ public: /// Used by joinGet function that turns StorageJoin into a dictionary. ColumnWithTypeAndName joinGet(const Block & block, const Block & block_with_columns_to_add) const; - /** Keep "totals" (separate part of dataset, see WITH TOTALS) to use later. - */ - void setTotals(const Block & block) override { totals = block; } - const Block & getTotals() const override { return totals; } - bool isFilled() const override { return from_storage_join || data->type == Type::DICT; } /** For RIGHT and FULL JOINs. @@ -390,8 +389,6 @@ private: Poco::Logger * log; - Block totals; - /// Should be set via setLock to protect hash table from modification from StorageJoin /// If set HashJoin instance is not available for modification (addJoinedBlock) TableLockHolder storage_join_lock = nullptr; diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index 64b576d3b96..2cc4ce175ba 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -4,12 +4,12 @@ #include #include +#include #include namespace DB { -class Block; struct ExtraBlock; using ExtraBlockPtr = std::shared_ptr; @@ -33,12 +33,17 @@ public: /// Could be called from different threads in parallel. virtual void joinBlock(Block & block, std::shared_ptr & not_processed) = 0; - /// Set/Get totals for right table - virtual void setTotals(const Block & block) = 0; - virtual const Block & getTotals() const = 0; + /** Set/Get totals for right table + * Keep "totals" (separate part of dataset, see WITH TOTALS) to use later. + */ + virtual void setTotals(const Block & block) { totals = block; } + virtual const Block & getTotals() const { return totals; } + /// Number of rows/bytes stored in memory virtual size_t getTotalRowCount() const = 0; virtual size_t getTotalByteCount() const = 0; + + /// Returns true if no data to join with. virtual bool alwaysReturnsEmptySet() const = 0; /// StorageJoin/Dictionary is already filled. No need to call addJoinedBlock. @@ -50,6 +55,9 @@ public: virtual std::shared_ptr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const = 0; + +private: + Block totals; }; using JoinPtr = std::shared_ptr; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index e6bd81b93fe..902d9f64058 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -92,12 +92,12 @@ public: BlockIO execute() override; /// Builds QueryPlan for current query. - virtual void buildQueryPlan(QueryPlan & query_plan) override; + void buildQueryPlan(QueryPlan & query_plan) override; bool ignoreLimits() const override { return options.ignore_limits; } bool ignoreQuota() const override { return options.ignore_quota; } - virtual void ignoreWithTotals() override; + void ignoreWithTotals() override; ASTPtr getQuery() const { return query_ptr; } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index 0e59e1e01ad..ff763ec6490 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -31,7 +31,7 @@ public: ~InterpreterSelectWithUnionQuery() override; /// Builds QueryPlan for current query. - virtual void buildQueryPlan(QueryPlan & query_plan) override; + void buildQueryPlan(QueryPlan & query_plan) override; BlockIO execute() override; @@ -43,7 +43,7 @@ public: ContextPtr context_, bool is_subquery = false); - virtual void ignoreWithTotals() override; + void ignoreWithTotals() override; bool supportsTransactions() const override { return true; } diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 482a813bfef..df47e8acdca 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -292,6 +294,7 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se return {}; auto settings = context->getSettingsRef(); + MultiEnum join_algorithm = settings.join_algorithm; auto table_join = std::make_shared(settings, context->getTemporaryVolume()); const ASTTablesInSelectQueryElement * ast_join = select_query.join(); @@ -305,9 +308,18 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se if (storage) { if (auto storage_join = std::dynamic_pointer_cast(storage); storage_join) + { table_join->setStorageJoin(storage_join); + } else if (auto storage_dict = std::dynamic_pointer_cast(storage); storage_dict) + { table_join->setStorageJoin(storage_dict); + } + else if (auto storage_kv = std::dynamic_pointer_cast(storage); + storage_kv && join_algorithm.isSet(JoinAlgorithm::DIRECT)) + { + table_join->setStorageJoin(storage_kv); + } } } diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 9ddd4ac0be0..1dea769f724 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -563,7 +563,7 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right /// Has to be called even if totals are empty void MergeJoin::setTotals(const Block & totals_block) { - totals = totals_block; + IJoin::setTotals(totals_block); mergeRightBlocks(); if (is_right || is_full) diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 89570015d0f..ab36599e6f4 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -29,7 +29,6 @@ public: void joinBlock(Block &, ExtraBlockPtr & not_processed) override; void setTotals(const Block &) override; - const Block & getTotals() const override { return totals; } size_t getTotalRowCount() const override { return right_blocks.row_count; } size_t getTotalByteCount() const override { return right_blocks.bytes; } @@ -100,7 +99,6 @@ private: std::unique_ptr disk_writer; /// Set of files with sorted blocks SortedBlocksWriter::SortedFiles flushed_right_blocks; - Block totals; std::atomic is_in_memory{true}; const bool is_any_join; const bool is_all_join; diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index fe204b34876..2683a5f7955 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace DB @@ -64,13 +65,7 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const // The user might add some ints values, and we will have Int Field, and the // insert will fail because the column requires Strings. Convert the fields // here, because it's hard to remember to convert them in all other places. - - Map map(attribute_names.size()); - for (size_t attr_idx = 0; attr_idx < map.size(); ++attr_idx) - { - map[attr_idx] = Tuple{attribute_names[attr_idx], toString(attribute_values[attr_idx])}; - } - columns[i++]->insert(map); + columns[i++]->insert(attributes); } @@ -158,8 +153,7 @@ void OpenTelemetrySpanHolder::addAttribute(const std::string& name, UInt64 value if (trace_id == UUID()) return; - this->attribute_names.push_back(name); - this->attribute_values.push_back(std::to_string(value)); + this->attributes.push_back(Tuple{name, toString(value)}); } void OpenTelemetrySpanHolder::addAttribute(const std::string& name, const std::string& value) @@ -167,8 +161,7 @@ void OpenTelemetrySpanHolder::addAttribute(const std::string& name, const std::s if (trace_id == UUID()) return; - this->attribute_names.push_back(name); - this->attribute_values.push_back(value); + this->attributes.push_back(Tuple{name, value}); } void OpenTelemetrySpanHolder::addAttribute(const Exception & e) @@ -176,8 +169,7 @@ void OpenTelemetrySpanHolder::addAttribute(const Exception & e) if (trace_id == UUID()) return; - this->attribute_names.push_back("clickhouse.exception"); - this->attribute_values.push_back(getExceptionMessage(e, false)); + this->attributes.push_back(Tuple{"clickhouse.exception", getExceptionMessage(e, false)}); } void OpenTelemetrySpanHolder::addAttribute(std::exception_ptr e) @@ -185,8 +177,7 @@ void OpenTelemetrySpanHolder::addAttribute(std::exception_ptr e) if (trace_id == UUID() || e == nullptr) return; - this->attribute_names.push_back("clickhouse.exception"); - this->attribute_values.push_back(getExceptionMessage(e, false)); + this->attributes.push_back(Tuple{"clickhouse.exception", getExceptionMessage(e, false)}); } bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & traceparent, diff --git a/src/Interpreters/OpenTelemetrySpanLog.h b/src/Interpreters/OpenTelemetrySpanLog.h index 677a283bb56..34f4765c8c4 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.h +++ b/src/Interpreters/OpenTelemetrySpanLog.h @@ -15,8 +15,7 @@ struct OpenTelemetrySpan std::string operation_name; UInt64 start_time_us; UInt64 finish_time_us; - Array attribute_names; - Array attribute_values; + Map attributes; // I don't understand how Links work, namely, which direction should they // point to, and how they are related with parent_span_id, so no Links for now. }; diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 3caf144d9a8..c2a6f513224 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -136,7 +136,8 @@ SortedBlocksWriter::TmpFilePtr SortedBlocksWriter::flush(const BlocksList & bloc pipeline.getHeader(), pipeline.getNumStreams(), sort_description, - rows_in_block); + rows_in_block, + SortingQueueStrategy::Default); pipeline.addTransform(std::move(transform)); } @@ -190,7 +191,8 @@ SortedBlocksWriter::PremergedFiles SortedBlocksWriter::premerge() pipeline.getHeader(), pipeline.getNumStreams(), sort_description, - rows_in_block); + rows_in_block, + SortingQueueStrategy::Default); pipeline.addTransform(std::move(transform)); } @@ -222,7 +224,8 @@ SortedBlocksWriter::SortedFiles SortedBlocksWriter::finishMerge(std::functionsecond; + return column_name; +} + NamesWithAliases TableJoin::getNamesWithAliases(const NameSet & required_columns) const { NamesWithAliases out; - for (const auto & column : required_columns) + out.reserve(required_columns.size()); + for (const auto & name : required_columns) { - auto it = original_names.find(column); - if (it != original_names.end()) - out.emplace_back(it->second, it->first); /// {original_name, name} + auto original_name = getOriginalName(name); + out.emplace_back(original_name, name); } return out; } @@ -513,6 +521,7 @@ TableJoin::createConvertingActions(const ColumnsWithTypeAndName & left_sample_co { if (dag) { + /// Just debug message std::vector input_cols; for (const auto & col : dag->getRequiredColumns()) input_cols.push_back(col.name + ": " + col.type->getName()); @@ -591,15 +600,16 @@ void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig catch (DB::Exception & ex) { throw DB::Exception(ErrorCodes::TYPE_MISMATCH, - "Can't infer common type for joined columns: {}: {} at left, {}: {} at right. {}", + "Can't infer common type for joined columns: {}: {} at left, {}: {} at right ({})", left_key_name, ltype->second->getName(), right_key_name, rtype->second->getName(), ex.message()); } - if (!allow_right && !common_type->equals(*rtype->second)) + bool right_side_changed = !common_type->equals(*rtype->second); + if (right_side_changed && !allow_right) { throw DB::Exception(ErrorCodes::TYPE_MISMATCH, - "Can't change type for right table: {}: {} -> {}.", + "Can't change type for right table: {}: {} -> {}", right_key_name, rtype->second->getName(), common_type->getName()); } left_type_map[left_key_name] = right_type_map[right_key_name] = common_type; @@ -626,7 +636,7 @@ static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src, bool has_some_to_do = false; for (auto & col : cols_dst) { - if (auto it = type_mapping.find(col.name); it != type_mapping.end()) + if (auto it = type_mapping.find(col.name); it != type_mapping.end() && col.type != it->second) { col.type = it->second; col.column = nullptr; @@ -635,6 +645,7 @@ static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src, } if (!has_some_to_do) return nullptr; + return ActionsDAG::makeConvertingActions(cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, add_new_cols, &key_column_rename); } @@ -685,6 +696,11 @@ ActionsDAGPtr TableJoin::applyKeyConvertToTable( return dag_stage1; } +void TableJoin::setStorageJoin(std::shared_ptr storage) +{ + right_kv_storage = storage; +} + void TableJoin::setStorageJoin(std::shared_ptr storage) { if (right_storage_dictionary) @@ -784,7 +800,7 @@ void TableJoin::resetToCross() bool TableJoin::allowParallelHashJoin() const { - if (dictionary_reader || join_algorithm != JoinAlgorithm::PARALLEL_HASH) + if (dictionary_reader || !join_algorithm.isSet(JoinAlgorithm::PARALLEL_HASH)) return false; if (table_join.kind != ASTTableJoin::Kind::Left && table_join.kind != ASTTableJoin::Kind::Inner) return false; diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 37e9417bde7..ff8c62100b6 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -9,7 +9,8 @@ #include #include #include -#include +#include + #include #include @@ -31,6 +32,7 @@ class Block; class DictionaryReader; class StorageJoin; class StorageDictionary; +class IKeyValueStorage; struct ColumnWithTypeAndName; using ColumnsWithTypeAndName = std::vector; @@ -55,7 +57,7 @@ public: struct JoinOnClause { Names key_names_left; - Names key_names_right; /// Duplicating right key names are qualified. + Names key_names_right; /// Duplicating right key names are qualified ASTPtr on_filter_condition_left; ASTPtr on_filter_condition_right; @@ -107,7 +109,7 @@ private: const size_t default_max_bytes = 0; const bool join_use_nulls = false; const size_t max_joined_block_rows = 0; - JoinAlgorithm join_algorithm = JoinAlgorithm::AUTO; + MultiEnum join_algorithm = MultiEnum(JoinAlgorithm::AUTO); const size_t partial_merge_join_rows_in_right_blocks = 0; const size_t partial_merge_join_left_table_buffer_bytes = 0; const size_t max_files_to_merge = 0; @@ -148,6 +150,8 @@ private: std::shared_ptr right_storage_dictionary; std::shared_ptr dictionary_reader; + std::shared_ptr right_kv_storage; + Names requiredJoinedNames() const; /// Create converting actions and change key column names if required @@ -189,14 +193,20 @@ public: const SizeLimits & sizeLimits() const { return size_limits; } VolumePtr getTemporaryVolume() { return tmp_volume; } bool allowMergeJoin() const; - bool preferMergeJoin() const { return join_algorithm == JoinAlgorithm::PREFER_PARTIAL_MERGE; } - bool forceMergeJoin() const { return join_algorithm == JoinAlgorithm::PARTIAL_MERGE; } - bool allowParallelHashJoin() const; + + bool isAllowedAlgorithm(JoinAlgorithm val) const { return join_algorithm.isSet(val) || join_algorithm.isSet(JoinAlgorithm::AUTO); } + bool isForcedAlgorithm(JoinAlgorithm val) const { return join_algorithm == MultiEnum(val); } + + bool preferMergeJoin() const { return join_algorithm == MultiEnum(JoinAlgorithm::PREFER_PARTIAL_MERGE); } + bool forceMergeJoin() const { return join_algorithm == MultiEnum(JoinAlgorithm::PARTIAL_MERGE); } bool forceHashJoin() const { /// HashJoin always used for DictJoin - return dictionary_reader || join_algorithm == JoinAlgorithm::HASH || join_algorithm == JoinAlgorithm::PARALLEL_HASH; + return dictionary_reader + || join_algorithm == MultiEnum(JoinAlgorithm::HASH) + || join_algorithm == MultiEnum(JoinAlgorithm::PARALLEL_HASH); } + bool allowParallelHashJoin() const; bool forceNullableRight() const { return join_use_nulls && isLeftOrFull(table_join.kind); } bool forceNullableLeft() const { return join_use_nulls && isRightOrFull(table_join.kind); } @@ -243,6 +253,7 @@ public: bool hasUsing() const { return table_join.using_expression_list != nullptr; } bool hasOn() const { return table_join.on_expression != nullptr; } + String getOriginalName(const String & column_name) const; NamesWithAliases getNamesWithAliases(const NameSet & required_columns) const; NamesWithAliases getRequiredColumns(const Block & sample, const Names & action_required_columns) const; @@ -294,6 +305,7 @@ public: std::unordered_map leftToRightKeyRemap() const; + void setStorageJoin(std::shared_ptr storage); void setStorageJoin(std::shared_ptr storage); void setStorageJoin(std::shared_ptr storage); @@ -301,8 +313,10 @@ public: bool tryInitDictJoin(const Block & sample_block, ContextPtr context); - bool isSpecialStorage() const { return right_storage_dictionary || right_storage_join; } + bool isSpecialStorage() const { return right_storage_dictionary || right_storage_join || right_kv_storage; } const DictionaryReader * getDictionaryReader() const { return dictionary_reader.get(); } + + std::shared_ptr getStorageKeyValue() { return right_kv_storage; } }; } diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 42db91f47c0..53d7fd0457a 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -384,8 +384,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) span.finish_time_us = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); - span.attribute_names.push_back("clickhouse.thread_id"); - span.attribute_values.push_back(thread_id); + span.attributes.push_back(Tuple{"clickhouse.thread_id", toString(thread_id)}); opentelemetry_span_log->add(span); } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 4b328f0466e..85c4ea261a0 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -301,28 +301,15 @@ static void onExceptionBeforeStart(const String & query_for_logging, ContextPtr span.operation_name = "query"; span.start_time_us = current_time_us; span.finish_time_us = time_in_microseconds(std::chrono::system_clock::now()); - - /// Keep values synchronized to type enum in QueryLogElement::createBlock. - span.attribute_names.push_back("clickhouse.query_status"); - span.attribute_values.push_back("ExceptionBeforeStart"); - - span.attribute_names.push_back("db.statement"); - span.attribute_values.push_back(elem.query); - - span.attribute_names.push_back("clickhouse.query_id"); - span.attribute_values.push_back(elem.client_info.current_query_id); - - span.attribute_names.push_back("clickhouse.exception"); - span.attribute_values.push_back(elem.exception); - - span.attribute_names.push_back("clickhouse.exception_code"); - span.attribute_values.push_back(elem.exception_code); - + span.attributes.reserve(6); + span.attributes.push_back(Tuple{"clickhouse.query_status", "ExceptionBeforeStart"}); + span.attributes.push_back(Tuple{"db.statement", elem.query}); + span.attributes.push_back(Tuple{"clickhouse.query_id", elem.client_info.current_query_id}); + span.attributes.push_back(Tuple{"clickhouse.exception", elem.exception}); + span.attributes.push_back(Tuple{"clickhouse.exception_code", toString(elem.exception_code)}); if (!context->query_trace_context.tracestate.empty()) { - span.attribute_names.push_back("clickhouse.tracestate"); - span.attribute_values.push_back( - context->query_trace_context.tracestate); + span.attributes.push_back(Tuple{"clickhouse.tracestate", context->query_trace_context.tracestate}); } opentelemetry_span_log->add(span); @@ -956,20 +943,13 @@ static std::tuple executeQueryImpl( span.start_time_us = elem.query_start_time_microseconds; span.finish_time_us = time_in_microseconds(finish_time); - /// Keep values synchronized to type enum in QueryLogElement::createBlock. - span.attribute_names.push_back("clickhouse.query_status"); - span.attribute_values.push_back("QueryFinish"); - - span.attribute_names.push_back("db.statement"); - span.attribute_values.push_back(elem.query); - - span.attribute_names.push_back("clickhouse.query_id"); - span.attribute_values.push_back(elem.client_info.current_query_id); + span.attributes.reserve(4); + span.attributes.push_back(Tuple{"clickhouse.query_status", "QueryFinish"}); + span.attributes.push_back(Tuple{"db.statement", elem.query}); + span.attributes.push_back(Tuple{"clickhouse.query_id", elem.client_info.current_query_id}); if (!context->query_trace_context.tracestate.empty()) { - span.attribute_names.push_back("clickhouse.tracestate"); - span.attribute_values.push_back( - context->query_trace_context.tracestate); + span.attributes.push_back(Tuple{"clickhouse.tracestate", context->query_trace_context.tracestate}); } opentelemetry_span_log->add(span); diff --git a/src/Interpreters/getClusterName.cpp b/src/Interpreters/getClusterName.cpp index fee10e32d70..d3c53b28cdf 100644 --- a/src/Interpreters/getClusterName.cpp +++ b/src/Interpreters/getClusterName.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,7 @@ std::string getClusterName(const IAST & node) return ast_id->name(); if (const auto * ast_lit = node.as()) - return ast_lit->value.safeGet(); + return checkAndGetLiteralArgument(*ast_lit, "cluster_name"); /// A hack to support hyphens in cluster names. if (const auto * ast_func = node.as()) diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 956f07811ae..4a8c9c14ea9 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -248,7 +248,7 @@ public: return removeOnCluster(clone(), params.default_database); } - virtual QueryKind getQueryKind() const override { return QueryKind::Alter; } + QueryKind getQueryKind() const override { return QueryKind::Alter; } protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index f9f57183a64..f3729b1523f 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -120,7 +120,7 @@ public: bool isView() const { return is_ordinary_view || is_materialized_view || is_live_view || is_window_view; } - virtual QueryKind getQueryKind() const override { return QueryKind::Create; } + QueryKind getQueryKind() const override { return QueryKind::Create; } protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index b4c96353a09..05515ba4005 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -45,7 +45,7 @@ public: return removeOnCluster(clone(), params.default_database); } - virtual QueryKind getQueryKind() const override { return QueryKind::Drop; } + QueryKind getQueryKind() const override { return QueryKind::Drop; } protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index db9262ea794..43780e27114 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -66,7 +66,7 @@ public: return res; } - virtual QueryKind getQueryKind() const override { return QueryKind::Insert; } + QueryKind getQueryKind() const override { return QueryKind::Insert; } protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTRenameQuery.h b/src/Parsers/ASTRenameQuery.h index ee7cad2d38a..723f680b492 100644 --- a/src/Parsers/ASTRenameQuery.h +++ b/src/Parsers/ASTRenameQuery.h @@ -65,7 +65,7 @@ public: return query_ptr; } - virtual QueryKind getQueryKind() const override { return QueryKind::Rename; } + QueryKind getQueryKind() const override { return QueryKind::Rename; } protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override diff --git a/src/Parsers/ASTSelectIntersectExceptQuery.h b/src/Parsers/ASTSelectIntersectExceptQuery.h index fa574b46c8d..c95944a0c35 100644 --- a/src/Parsers/ASTSelectIntersectExceptQuery.h +++ b/src/Parsers/ASTSelectIntersectExceptQuery.h @@ -22,7 +22,7 @@ public: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; - virtual QueryKind getQueryKind() const override { return QueryKind::SelectIntersectExcept; } + QueryKind getQueryKind() const override { return QueryKind::SelectIntersectExcept; } ASTs getListOfSelects() const; diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index 084627b57a1..1a02717db8d 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -140,7 +140,7 @@ public: void setFinal(); - virtual QueryKind getQueryKind() const override { return QueryKind::Select; } + QueryKind getQueryKind() const override { return QueryKind::Select; } protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index bd45dd7fc05..457a3361b1e 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -17,7 +17,7 @@ public: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; - virtual QueryKind getQueryKind() const override { return QueryKind::Select; } + QueryKind getQueryKind() const override { return QueryKind::Select; } SelectUnionMode union_mode; diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 6bcd5f090d0..f2ef7a7a47a 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -114,7 +114,7 @@ public: return removeOnCluster(clone(), params.default_database); } - virtual QueryKind getQueryKind() const override { return QueryKind::System; } + QueryKind getQueryKind() const override { return QueryKind::System; } protected: diff --git a/src/Parsers/ASTTablesInSelectQuery.h b/src/Parsers/ASTTablesInSelectQuery.h index 500337936d1..2b07d31fb7d 100644 --- a/src/Parsers/ASTTablesInSelectQuery.h +++ b/src/Parsers/ASTTablesInSelectQuery.h @@ -121,6 +121,7 @@ inline bool isCrossOrComma(ASTTableJoin::Kind kind) { return kind == ASTTableJoi inline bool isRightOrFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full; } inline bool isLeftOrFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full; } inline bool isInnerOrRight(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Right; } +inline bool isInnerOrLeft(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Left; } /// Specification of ARRAY JOIN. diff --git a/src/Parsers/Access/ASTGrantQuery.h b/src/Parsers/Access/ASTGrantQuery.h index d2df3f6cf6e..8c7df3cd57e 100644 --- a/src/Parsers/Access/ASTGrantQuery.h +++ b/src/Parsers/Access/ASTGrantQuery.h @@ -34,6 +34,6 @@ public: void replaceEmptyDatabase(const String & current_database); void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams &) const override { return removeOnCluster(clone()); } - virtual QueryKind getQueryKind() const override { return is_revoke ? QueryKind::Revoke : QueryKind::Grant; } + QueryKind getQueryKind() const override { return is_revoke ? QueryKind::Revoke : QueryKind::Grant; } }; } diff --git a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt index 3d416544419..86eb8bf36a5 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt +++ b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt @@ -31,7 +31,8 @@ add_custom_command( DEPENDS "${CURRENT_DIR_IN_BINARY}/clickhouse.g" ) -PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS "${CURRENT_DIR_IN_BINARY}/out.proto") +set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) +protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS "${CURRENT_DIR_IN_BINARY}/out.proto") set(FUZZER_SRCS codegen_select_fuzzer.cpp "${CURRENT_DIR_IN_BINARY}/out.cpp" ${PROTO_SRCS} ${PROTO_HDRS}) set(CMAKE_INCLUDE_CURRENT_DIR TRUE) diff --git a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp new file mode 100644 index 00000000000..749b4b40984 --- /dev/null +++ b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp @@ -0,0 +1,102 @@ +#include +#include + + +namespace DB +{ + +SQLInsertRowOutputFormat::SQLInsertRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_) + : IRowOutputFormat(header_, out_, params_), column_names(header_.getNames()), format_settings(format_settings_) +{ +} + +void SQLInsertRowOutputFormat::writeRowStartDelimiter() +{ + if (rows_in_line == 0) + printLineStart(); + writeChar('(', out); +} + +void SQLInsertRowOutputFormat::printLineStart() +{ + if (format_settings.sql_insert.use_replace) + writeCString("REPLACE INTO ", out); + else + writeCString("INSERT INTO ", out); + + writeString(format_settings.sql_insert.table_name, out); + + if (format_settings.sql_insert.include_column_names) + printColumnNames(); + + writeCString(" VALUES ", out); +} + +void SQLInsertRowOutputFormat::printColumnNames() +{ + writeCString(" (", out); + for (size_t i = 0; i != column_names.size(); ++i) + { + if (format_settings.sql_insert.quote_names) + writeChar('`', out); + + writeString(column_names[i], out); + + if (format_settings.sql_insert.quote_names) + writeChar('`', out); + + if (i + 1 != column_names.size()) + writeCString(", ", out); + } + writeChar(')', out); +} + +void SQLInsertRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) +{ + serialization.serializeTextQuoted(column, row_num, out, format_settings); +} + +void SQLInsertRowOutputFormat::writeFieldDelimiter() +{ + writeCString(", ", out); +} + +void SQLInsertRowOutputFormat::writeRowEndDelimiter() +{ + writeChar(')', out); + ++rows_in_line; +} + +void SQLInsertRowOutputFormat::writeRowBetweenDelimiter() +{ + if (rows_in_line >= format_settings.sql_insert.max_batch_size) + { + writeCString(";\n", out); + rows_in_line = 0; + } + else + { + writeCString(", ", out); + } +} + +void SQLInsertRowOutputFormat::writeSuffix() +{ + writeCString(";\n", out); +} + + +void registerOutputFormatSQLInsert(FormatFactory & factory) +{ + factory.registerOutputFormat("SQLInsert", []( + WriteBuffer & buf, + const Block & sample, + const RowOutputFormatParams & params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, params, settings); + }); +} + + +} diff --git a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h new file mode 100644 index 00000000000..aaaf39a9e4d --- /dev/null +++ b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; + +class SQLInsertRowOutputFormat : public IRowOutputFormat +{ +public: + SQLInsertRowOutputFormat( + WriteBuffer & out_, + const Block & header_, + const RowOutputFormatParams & params_, + const FormatSettings & format_settings_); + + String getName() const override { return "SQLInsertRowOutputFormat"; } + + /// https://www.iana.org/assignments/media-types/text/tab-separated-values + String getContentType() const override { return "text/tab-separated-values; charset=UTF-8"; } + +protected: + void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; + virtual void writeFieldDelimiter() override; + virtual void writeRowStartDelimiter() override; + virtual void writeRowEndDelimiter() override; + virtual void writeRowBetweenDelimiter() override; + virtual void writeSuffix() override; + + void printLineStart(); + void printColumnNames(); + + size_t rows_in_line = 0; + Names column_names; + const FormatSettings format_settings; +}; + +} diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index d35d267731e..25719166acd 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -12,6 +12,7 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( size_t num_inputs, SortDescription description_, size_t max_block_size, + SortingQueueStrategy sorting_queue_strategy_, UInt64 limit_, WriteBuffer * out_row_sources_buf_, bool use_average_block_sizes) @@ -21,6 +22,7 @@ MergingSortedAlgorithm::MergingSortedAlgorithm( , limit(limit_) , out_row_sources_buf(out_row_sources_buf_) , current_inputs(num_inputs) + , sorting_queue_strategy(sorting_queue_strategy_) , cursors(num_inputs) { DataTypes sort_description_types; @@ -69,11 +71,22 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) cursors[source_num] = SortCursorImpl(header, chunk.getColumns(), description, source_num); } - queue_variants.callOnBatchVariant([&](auto & queue) + if (sorting_queue_strategy == SortingQueueStrategy::Default) { - using QueueType = std::decay_t; - queue = QueueType(cursors); - }); + queue_variants.callOnVariant([&](auto & queue) + { + using QueueType = std::decay_t; + queue = QueueType(cursors); + }); + } + else + { + queue_variants.callOnBatchVariant([&](auto & queue) + { + using QueueType = std::decay_t; + queue = QueueType(cursors); + }); + } } void MergingSortedAlgorithm::consume(Input & input, size_t source_num) @@ -82,14 +95,34 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) current_inputs[source_num].swap(input); cursors[source_num].reset(current_inputs[source_num].chunk.getColumns(), header); - queue_variants.callOnBatchVariant([&](auto & queue) + if (sorting_queue_strategy == SortingQueueStrategy::Default) { - queue.push(cursors[source_num]); - }); + queue_variants.callOnVariant([&](auto & queue) + { + queue.push(cursors[source_num]); + }); + } + else + { + queue_variants.callOnBatchVariant([&](auto & queue) + { + queue.push(cursors[source_num]); + }); + } } IMergingAlgorithm::Status MergingSortedAlgorithm::merge() { + if (sorting_queue_strategy == SortingQueueStrategy::Default) + { + IMergingAlgorithm::Status result = queue_variants.callOnVariant([&](auto & queue) + { + return mergeImpl(queue); + }); + + return result; + } + IMergingAlgorithm::Status result = queue_variants.callOnBatchVariant([&](auto & queue) { return mergeBatchImpl(queue); @@ -98,6 +131,100 @@ IMergingAlgorithm::Status MergingSortedAlgorithm::merge() return result; } +template +IMergingAlgorithm::Status MergingSortedAlgorithm::mergeImpl(TSortingHeap & queue) +{ + /// Take rows in required order and put them into `merged_data`, while the rows are no more than `max_block_size` + while (queue.isValid()) + { + if (merged_data.hasEnoughRows()) + return Status(merged_data.pull()); + + auto current = queue.current(); + + if (current.impl->isLast() && current_inputs[current.impl->order].skip_last_row) + { + /// Get the next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + + if (current.impl->isFirst() + && !current_inputs[current.impl->order].skip_last_row /// Ignore optimization if last row should be skipped. + && (queue.size() == 1 + || (queue.size() >= 2 && current.totallyLessOrEquals(queue.nextChild())))) + { + /** This is special optimization if current cursor is totally less than next cursor. + * We want to insert current cursor chunk directly in merged data. + * + * First if merged_data is not empty we need to flush it. + * We will get into the same condition on next mergeBatch call. + * + * Then we can insert chunk directly in merged data. + */ + + if (merged_data.mergedRows() != 0) + return Status(merged_data.pull()); + + /// Actually, current.impl->order stores source number (i.e. cursors[current.impl->order] == current.impl) + size_t source_num = current.impl->order; + + auto chunk_num_rows = current_inputs[source_num].chunk.getNumRows(); + + UInt64 total_merged_rows_after_insertion = merged_data.mergedRows() + chunk_num_rows; + bool limit_reached = limit && total_merged_rows_after_insertion >= limit; + + if (limit && total_merged_rows_after_insertion > limit) + chunk_num_rows -= total_merged_rows_after_insertion - limit; + + merged_data.insertChunk(std::move(current_inputs[source_num].chunk), chunk_num_rows); + current_inputs[source_num].chunk = Chunk(); + + /// Write order of rows for other columns this data will be used in gather stream + if (out_row_sources_buf) + { + RowSourcePart row_source(source_num); + for (size_t i = 0; i < chunk_num_rows; ++i) + out_row_sources_buf->write(row_source.data); + } + + /// We will get the next block from the corresponding source, if there is one. + queue.removeTop(); + + auto status = Status(merged_data.pull(), limit_reached); + + if (!limit_reached) + status.required_source = source_num; + + return status; + } + + merged_data.insertRow(current->all_columns, current->getRow(), current->rows); + + if (out_row_sources_buf) + { + RowSourcePart row_source(current.impl->order); + out_row_sources_buf->write(row_source.data); + } + + if (limit && merged_data.totalMergedRows() >= limit) + return Status(merged_data.pull(), true); + + if (!current->isLast()) + { + queue.next(); + } + else + { + /// We will get the next block from the corresponding source, if there is one. + queue.removeTop(); + return Status(current.impl->order); + } + } + + return Status(merged_data.pull(), true); +} + template IMergingAlgorithm::Status MergingSortedAlgorithm::mergeBatchImpl(TSortingQueue & queue) @@ -134,14 +261,22 @@ IMergingAlgorithm::Status MergingSortedAlgorithm::mergeBatchImpl(TSortingQueue & } bool limit_reached = false; - if (limit && merged_rows + updated_batch_size > limit) + + if (limit && merged_rows + updated_batch_size >= limit && !batch_skip_last_row) + { + updated_batch_size -= merged_rows + updated_batch_size - limit; + limit_reached = true; + } + else if (limit && merged_rows + updated_batch_size > limit) { batch_skip_last_row = false; updated_batch_size -= merged_rows + updated_batch_size - limit; limit_reached = true; } - if (unlikely(current.impl->isFirst() && current.impl->isLast(initial_batch_size))) + if (unlikely(current.impl->isFirst() && + current.impl->isLast(initial_batch_size) && + !current_inputs[current.impl->order].skip_last_row)) { /** This is special optimization if current cursor is totally less than next cursor. * We want to insert current cursor chunk directly in merged data. @@ -167,9 +302,6 @@ IMergingAlgorithm::Status MergingSortedAlgorithm::mergeBatchImpl(TSortingQueue & out_row_sources_buf->write(row_source.data); } - if (limit_reached) - break; - /// We will get the next block from the corresponding source, if there is one. queue.removeTop(); diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h index 9e517120f38..cf32e5fd4dd 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h @@ -18,6 +18,7 @@ public: size_t num_inputs, SortDescription description_, size_t max_block_size, + SortingQueueStrategy sorting_queue_strategy_, UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, bool use_average_block_sizes = false); @@ -47,10 +48,15 @@ private: /// Chunks currently being merged. Inputs current_inputs; + SortingQueueStrategy sorting_queue_strategy; + SortCursorImpls cursors; SortQueueVariants queue_variants; + template + Status mergeImpl(TSortingQueue & queue); + template Status mergeBatchImpl(TSortingQueue & queue); diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index c244388464f..4cb74ffc71e 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -12,6 +12,7 @@ MergingSortedTransform::MergingSortedTransform( size_t num_inputs, SortDescription description_, size_t max_block_size, + SortingQueueStrategy sorting_queue_strategy, UInt64 limit_, WriteBuffer * out_row_sources_buf_, bool quiet_, @@ -23,6 +24,7 @@ MergingSortedTransform::MergingSortedTransform( num_inputs, std::move(description_), max_block_size, + sorting_queue_strategy, limit_, out_row_sources_buf_, use_average_block_sizes) diff --git a/src/Processors/Merges/MergingSortedTransform.h b/src/Processors/Merges/MergingSortedTransform.h index 93bd36d8aec..16e3e2791ee 100644 --- a/src/Processors/Merges/MergingSortedTransform.h +++ b/src/Processors/Merges/MergingSortedTransform.h @@ -16,6 +16,7 @@ public: size_t num_inputs, SortDescription description, size_t max_block_size, + SortingQueueStrategy sorting_queue_strategy, UInt64 limit_ = 0, WriteBuffer * out_row_sources_buf_ = nullptr, bool quiet_ = false, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 4a1772759bc..b4e143cc002 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -583,7 +583,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( pipe.getHeader(), pipe.numOutputPorts(), sort_description, - max_block_size); + max_block_size, + SortingQueueStrategy::Batch); pipe.addTransform(std::move(transform)); } @@ -611,7 +612,7 @@ static void addMergingFinal( { case MergeTreeData::MergingParams::Ordinary: return std::make_shared(header, num_outputs, - sort_description, max_block_size); + sort_description, max_block_size, SortingQueueStrategy::Batch); case MergeTreeData::MergingParams::Collapsing: return std::make_shared(header, num_outputs, diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 89a5ecb4cfa..46588ada225 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -124,6 +124,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getNumStreams(), prefix_description, max_block_size, + SortingQueueStrategy::Batch, limit_for_merging); pipeline.addTransform(std::move(transform)); @@ -212,6 +213,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getNumStreams(), result_description, max_block_size, + SortingQueueStrategy::Batch, limit); pipeline.addTransform(std::move(transform)); @@ -226,6 +228,7 @@ void SortingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build pipeline.getNumStreams(), result_description, max_block_size, + SortingQueueStrategy::Batch, limit); pipeline.addTransform(std::move(transform)); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index f8332742978..7f5896f5e97 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -501,6 +501,8 @@ void AggregatingTransform::work() Processors AggregatingTransform::expandPipeline() { + if (processors.empty()) + throw Exception("Can not expandPipeline in AggregatingTransform. This is a bug.", ErrorCodes::LOGICAL_ERROR); auto & out = processors.back()->getOutputs().front(); inputs.emplace_back(out.getHeader(), this); connect(out, inputs.back()); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 0c4615e9273..7c0422584c9 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -203,6 +203,7 @@ void MergeSortingTransform::consume(Chunk chunk) 0, description, max_merged_block_size, + SortingQueueStrategy::Batch, limit, nullptr, quiet, diff --git a/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp b/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp index f9eca5f1ee0..2fa5873544f 100644 --- a/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp +++ b/src/QueryPipeline/tests/gtest_blocks_size_merging_streams.cpp @@ -83,7 +83,7 @@ TEST(MergingSortedTest, SimpleBlockSizeTest) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, SortingQueueStrategy::Batch, 0, nullptr, false, true); pipe.addTransform(std::move(transform)); @@ -125,7 +125,7 @@ TEST(MergingSortedTest, MoreInterestingBlockSizes) EXPECT_EQ(pipe.numOutputPorts(), 3); auto transform = std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), sort_description, - DEFAULT_MERGE_BLOCK_SIZE, false, nullptr, false, true); + DEFAULT_MERGE_BLOCK_SIZE, SortingQueueStrategy::Batch, 0, nullptr, false, true); pipe.addTransform(std::move(transform)); diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index f916ac8c2af..0d6beb1733b 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -18,6 +18,9 @@ #if USE_MYSQL #include #endif +#if USE_NATSIO +#include +#endif #include @@ -542,6 +545,11 @@ template bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); #endif +#if USE_NATSIO +template +bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); +#endif + template std::optional getExternalDataSourceConfiguration( const ASTs & args, ContextPtr context, bool is_database_engine, bool throw_on_no_collection, const BaseSettings & storage_settings); diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 4bf77792559..323bcdc100d 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -9,16 +9,14 @@ #include #include #include -#include #include -#include #include #include #include -#include #include #include #include +#include #include #include #include @@ -805,8 +803,8 @@ void registerStorageFileLog(StorageFactory & factory) auto path_ast = evaluateConstantExpressionAsLiteral(engine_args[0], args.getContext()); auto format_ast = evaluateConstantExpressionAsLiteral(engine_args[1], args.getContext()); - auto path = path_ast->as().value.safeGet(); - auto format = format_ast->as().value.safeGet(); + auto path = checkAndGetLiteralArgument(path_ast, "path"); + auto format = checkAndGetLiteralArgument(format_ast, "format"); return std::make_shared( args.table_id, diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 708bfd5ef8b..5e811f8e42c 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -664,13 +665,13 @@ void registerStorageHDFS(StorageFactory & factory) engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.getLocalContext()); - String url = engine_args[0]->as().value.safeGet(); + String url = checkAndGetLiteralArgument(engine_args[0], "url"); String format_name = "auto"; if (engine_args.size() > 1) { engine_args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[1], args.getLocalContext()); - format_name = engine_args[1]->as().value.safeGet(); + format_name = checkAndGetLiteralArgument(engine_args[1], "format_name"); } if (format_name == "auto") @@ -680,7 +681,7 @@ void registerStorageHDFS(StorageFactory & factory) if (engine_args.size() == 3) { engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.getLocalContext()); - compression_method = engine_args[2]->as().value.safeGet(); + compression_method = checkAndGetLiteralArgument(engine_args[2], "compression_method"); } else compression_method = "auto"; ASTPtr partition_by; diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index b717d373598..b3e1ea93c94 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,8 @@ #include #include #include -#include +#include + namespace DB { @@ -959,9 +961,9 @@ void registerStorageHive(StorageFactory & factory) for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext()); - const String & hive_metastore_url = engine_args[0]->as().value.safeGet(); - const String & hive_database = engine_args[1]->as().value.safeGet(); - const String & hive_table = engine_args[2]->as().value.safeGet(); + const String & hive_metastore_url = checkAndGetLiteralArgument(engine_args[0], "hive_metastore_url"); + const String & hive_database = checkAndGetLiteralArgument(engine_args[1], "hive_database"); + const String & hive_table = checkAndGetLiteralArgument(engine_args[2], "hive_table"); return std::make_shared( hive_metastore_url, hive_database, diff --git a/src/Storages/IKVStorage.h b/src/Storages/IKVStorage.h new file mode 100644 index 00000000000..667ccda0c41 --- /dev/null +++ b/src/Storages/IKVStorage.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Storage that support key-value requests +class IKeyValueStorage : public IStorage +{ +public: + using IStorage::IStorage; + + /// Get primary key name that supports key-value requests. + /// Primary key can constist of multiple columns. + virtual Names getPrimaryKey() const = 0; + + /* + * Get data from storage directly by keys. + * + * @param keys - keys to get data for. Key can be compound and represented by several columns. + * @param out_null_map - output parameter indicating which keys were not found. + * + * @return - chunk of data corresponding for keys. + * Number of rows in chunk is equal to size of columns in keys. + * If the key was not found row would have a default value. + */ + virtual Chunk getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray & out_null_map) const = 0; +}; + +} diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index fc29769790d..8bbb2fa1c04 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -1,11 +1,9 @@ #include #include -#include #include #include #include -#include #include #include #include diff --git a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp index 02dca993436..c5966d9e322 100644 --- a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -14,7 +13,7 @@ #include #include #include -#include +#include #include #include @@ -156,11 +155,11 @@ MeiliSearchConfiguration StorageMeiliSearch::getConfiguration(ASTs engine_args, for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); - String url = engine_args[0]->as().value.safeGet(); - String index = engine_args[1]->as().value.safeGet(); + String url = checkAndGetLiteralArgument(engine_args[0], "url"); + String index = checkAndGetLiteralArgument(engine_args[1], "index"); String key; if (engine_args.size() == 3) - key = engine_args[2]->as().value.safeGet(); + key = checkAndGetLiteralArgument(engine_args[2], "key"); return MeiliSearchConfiguration(url, index, key); } } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index fc4a4554304..2cc28d020bb 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -861,7 +861,6 @@ std::string DataPartStorageBuilderOnDisk::getRelativePath() const void DataPartStorageBuilderOnDisk::createDirectories() { - LOG_INFO(&Poco::Logger::get("DEBUG"), "CREATING DIRECTORY {}", (fs::path(root_path) / part_dir).string()); transaction->createDirectories(fs::path(root_path) / part_dir); } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index f16d22f553a..7426b384394 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -855,7 +855,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() { case MergeTreeData::MergingParams::Ordinary: merged_transform = std::make_shared( - header, pipes.size(), sort_description, merge_block_size, 0, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); + header, pipes.size(), sort_description, merge_block_size, SortingQueueStrategy::Default, 0, ctx->rows_sources_write_buf.get(), true, ctx->blocks_are_granules_size); break; case MergeTreeData::MergingParams::Collapsing: diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 453b3c55961..e5dc11e9916 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5479,7 +5479,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (analysis_result.before_where) { candidate.where_column_name = analysis_result.where_column_name; - candidate.remove_where_filter = analysis_result.remove_where_filter; + candidate.remove_where_filter = !required_columns.contains(analysis_result.where_column_name); candidate.before_where = analysis_result.before_where->clone(); auto new_required_columns = candidate.before_where->foldActionsByProjection( @@ -5612,20 +5612,9 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg candidate.before_aggregation->reorderAggregationKeysForProjection(key_name_pos_map); candidate.before_aggregation->addAggregatesViaProjection(aggregates); - // minmax_count_projections only have aggregation actions - if (minmax_count_projection) - candidate.required_columns = {required_columns.begin(), required_columns.end()}; - if (rewrite_before_where(candidate, projection, required_columns, sample_block_for_keys, aggregates)) { - if (minmax_count_projection) - { - candidate.before_where = nullptr; - candidate.prewhere_info = nullptr; - } - else - candidate.required_columns = {required_columns.begin(), required_columns.end()}; - + candidate.required_columns = {required_columns.begin(), required_columns.end()}; for (const auto & aggregate : aggregates) candidate.required_columns.push_back(aggregate.name); candidates.push_back(std::move(candidate)); diff --git a/src/Storages/NATS/Buffer_fwd.h b/src/Storages/NATS/Buffer_fwd.h new file mode 100644 index 00000000000..3eb52314a79 --- /dev/null +++ b/src/Storages/NATS/Buffer_fwd.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace DB +{ + +class ReadBufferFromNATSConsumer; +using ConsumerBufferPtr = std::shared_ptr; + +class WriteBufferToNATSProducer; +using ProducerBufferPtr = std::shared_ptr; + +} diff --git a/src/Storages/NATS/NATSConnection.cpp b/src/Storages/NATS/NATSConnection.cpp new file mode 100644 index 00000000000..359754bb144 --- /dev/null +++ b/src/Storages/NATS/NATSConnection.cpp @@ -0,0 +1,151 @@ +#include "NATSConnection.h" + +#include +#include + +#include + + +namespace DB +{ + +//static const auto CONNECT_SLEEP = 200; +static const auto RETRIES_MAX = 20; +static const auto CONNECTED_TO_BUFFER_SIZE = 256; + + +NATSConnectionManager::NATSConnectionManager(const NATSConfiguration & configuration_, Poco::Logger * log_) + : configuration(configuration_) + , log(log_) + , event_handler(loop.getLoop(), log) +{ +} + + +NATSConnectionManager::~NATSConnectionManager() +{ + if (has_connection) + natsConnection_Destroy(connection); +} + +String NATSConnectionManager::connectionInfoForLog() const +{ + if (!configuration.url.empty()) + { + return "url : " + configuration.url; + } + return "cluster: " + boost::algorithm::join(configuration.servers, ", "); +} + +bool NATSConnectionManager::isConnected() +{ + std::lock_guard lock(mutex); + return isConnectedImpl(); +} + +bool NATSConnectionManager::connect() +{ + std::lock_guard lock(mutex); + connectImpl(); + return isConnectedImpl(); +} + +bool NATSConnectionManager::reconnect() +{ + std::lock_guard lock(mutex); + if (isConnectedImpl()) + return true; + + disconnectImpl(); + + LOG_DEBUG(log, "Trying to restore connection to NATS {}", connectionInfoForLog()); + connectImpl(); + + return isConnectedImpl(); +} + +void NATSConnectionManager::disconnect() +{ + std::lock_guard lock(mutex); + disconnectImpl(); +} + +bool NATSConnectionManager::closed() +{ + std::lock_guard lock(mutex); + return natsConnection_IsClosed(connection); +} + +bool NATSConnectionManager::isConnectedImpl() const +{ + return connection && has_connection && !natsConnection_IsClosed(connection); +} + +void NATSConnectionManager::connectImpl() +{ + natsOptions * options = event_handler.getOptions(); + if (!configuration.username.empty() && !configuration.password.empty()) + natsOptions_SetUserInfo(options, configuration.username.c_str(), configuration.password.c_str()); + if (!configuration.token.empty()) + natsOptions_SetToken(options, configuration.token.c_str()); + + if (configuration.secure) + { + natsOptions_SetSecure(options, true); + natsOptions_SkipServerVerification(options, true); + } + if (!configuration.url.empty()) + { + natsOptions_SetURL(options, configuration.url.c_str()); + } + else + { + const char * servers[configuration.servers.size()]; + for (size_t i = 0; i < configuration.servers.size(); ++i) + { + servers[i] = configuration.servers[i].c_str(); + } + natsOptions_SetServers(options, servers, configuration.servers.size()); + } + natsOptions_SetMaxReconnect(options, configuration.max_reconnect); + natsOptions_SetReconnectWait(options, configuration.reconnect_wait); + natsOptions_SetDisconnectedCB(options, disconnectedCallback, log); + natsOptions_SetReconnectedCB(options, reconnectedCallback, log); + natsStatus status; + { + auto lock = event_handler.setThreadLocalLoop(); + status = natsConnection_Connect(&connection, options); + } + if (status == NATS_OK) + has_connection = true; + else + LOG_DEBUG(log, "New connection to {} failed. Nats status text: {}. Last error message: {}", + connectionInfoForLog(), natsStatus_GetText(status), nats_GetLastError(nullptr)); +} + +void NATSConnectionManager::disconnectImpl() +{ + if (!has_connection) + return; + + natsConnection_Close(connection); + + size_t cnt_retries = 0; + while (!natsConnection_IsClosed(connection) && cnt_retries++ != RETRIES_MAX) + event_handler.iterateLoop(); +} + +void NATSConnectionManager::reconnectedCallback(natsConnection * nc, void * log) +{ + char buffer[CONNECTED_TO_BUFFER_SIZE]; + buffer[0] = '\0'; + natsConnection_GetConnectedUrl(nc, buffer, sizeof(buffer)); + LOG_DEBUG(static_cast(log), "Got reconnected to NATS server: {}.", buffer); +} + +void NATSConnectionManager::disconnectedCallback(natsConnection *, void * log) +{ + LOG_DEBUG(static_cast(log), "Got disconnected from NATS server."); +} + +} diff --git a/src/Storages/NATS/NATSConnection.h b/src/Storages/NATS/NATSConnection.h new file mode 100644 index 00000000000..78a273164db --- /dev/null +++ b/src/Storages/NATS/NATSConnection.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +struct NATSConfiguration +{ + String url; + std::vector servers; + + String username; + String password; + String token; + + int max_reconnect; + int reconnect_wait; + + bool secure; +}; + +class NATSConnectionManager +{ +public: + NATSConnectionManager(const NATSConfiguration & configuration_, Poco::Logger * log_); + ~NATSConnectionManager(); + + bool isConnected(); + + bool connect(); + + bool reconnect(); + + void disconnect(); + + bool closed(); + + /// NATSHandler is thread safe. Any public methods can be called concurrently. + NATSHandler & getHandler() { return event_handler; } + natsConnection * getConnection() { return connection; } + + String connectionInfoForLog() const; + +private: + bool isConnectedImpl() const; + + void connectImpl(); + + void disconnectImpl(); + + static void disconnectedCallback(natsConnection * nc, void * log); + static void reconnectedCallback(natsConnection * nc, void * log); + + NATSConfiguration configuration; + Poco::Logger * log; + + UVLoop loop; + NATSHandler event_handler; + + + natsConnection * connection; + // true if at any point successfully connected to NATS + bool has_connection = false; + + std::mutex mutex; +}; + +using NATSConnectionManagerPtr = std::shared_ptr; + +} diff --git a/src/Storages/NATS/NATSHandler.cpp b/src/Storages/NATS/NATSHandler.cpp new file mode 100644 index 00000000000..b5812bc3349 --- /dev/null +++ b/src/Storages/NATS/NATSHandler.cpp @@ -0,0 +1,83 @@ +#include +#include +#include +#include + +namespace DB +{ + +/* The object of this class is shared between concurrent consumers (who share the same connection == share the same + * event loop and handler). + */ + +static const auto MAX_THREAD_WORK_DURATION_MS = 60000; + +NATSHandler::NATSHandler(uv_loop_t * loop_, Poco::Logger * log_) : + loop(loop_), + log(log_), + loop_running(false), + loop_state(Loop::STOP) +{ + natsLibuv_Init(); + natsLibuv_SetThreadLocalLoop(loop); + natsOptions_Create(&opts); + natsOptions_SetEventLoop(opts, static_cast(loop), + natsLibuv_Attach, + natsLibuv_Read, + natsLibuv_Write, + natsLibuv_Detach); + natsOptions_SetIOBufSize(opts, INT_MAX); + natsOptions_SetSendAsap(opts, true); +} + +void NATSHandler::startLoop() +{ + std::lock_guard lock(startup_mutex); + natsLibuv_SetThreadLocalLoop(loop); + + LOG_DEBUG(log, "Background loop started"); + loop_running.store(true); + auto start_time = std::chrono::steady_clock::now(); + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time); + + while (loop_state.load() == Loop::RUN && duration.count() < MAX_THREAD_WORK_DURATION_MS) + { + uv_run(loop, UV_RUN_NOWAIT); + end_time = std::chrono::steady_clock::now(); + duration = std::chrono::duration_cast(end_time - start_time); + } + + LOG_DEBUG(log, "Background loop ended"); + loop_running.store(false); +} + +void NATSHandler::iterateLoop() +{ + std::unique_lock lock(startup_mutex, std::defer_lock); + if (lock.try_lock()) + { + natsLibuv_SetThreadLocalLoop(loop); + uv_run(loop, UV_RUN_NOWAIT); + } +} + +LockPtr NATSHandler::setThreadLocalLoop() +{ + auto lock = std::make_unique>(startup_mutex); + natsLibuv_SetThreadLocalLoop(loop); + return lock; +} + +void NATSHandler::stopLoop() +{ + LOG_DEBUG(log, "Implicit loop stop."); + uv_stop(loop); +} + +NATSHandler::~NATSHandler() +{ + natsOptions_Destroy(opts); +} + +} diff --git a/src/Storages/NATS/NATSHandler.h b/src/Storages/NATS/NATSHandler.h new file mode 100644 index 00000000000..e3894c888a3 --- /dev/null +++ b/src/Storages/NATS/NATSHandler.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace Loop +{ + static const UInt8 RUN = 1; + static const UInt8 STOP = 2; +} + +using SubscriptionPtr = std::unique_ptr; +using LockPtr = std::unique_ptr>; + +class NATSHandler +{ +public: + NATSHandler(uv_loop_t * loop_, Poco::Logger * log_); + + ~NATSHandler(); + + /// Loop for background thread worker. + void startLoop(); + + /// Loop to wait for small tasks in a non-blocking mode. + /// Adds synchronization with main background loop. + void iterateLoop(); + + LockPtr setThreadLocalLoop(); + + void stopLoop(); + bool loopRunning() const { return loop_running.load(); } + + void updateLoopState(UInt8 state) { loop_state.store(state); } + UInt8 getLoopState() { return loop_state.load(); } + + natsOptions * getOptions() { return opts; } + +private: + uv_loop_t * loop; + natsOptions * opts = nullptr; + Poco::Logger * log; + + std::atomic loop_running; + std::atomic loop_state; + std::mutex startup_mutex; +}; + +} diff --git a/src/Storages/NATS/NATSSettings.cpp b/src/Storages/NATS/NATSSettings.cpp new file mode 100644 index 00000000000..ffdb79247d2 --- /dev/null +++ b/src/Storages/NATS/NATSSettings.cpp @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_SETTING; +} + +IMPLEMENT_SETTINGS_TRAITS(NATSSettingsTraits, LIST_OF_NATS_SETTINGS) + +void NATSSettings::loadFromQuery(ASTStorage & storage_def) +{ + if (storage_def.settings) + { + try + { + applyChanges(storage_def.settings->changes); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_SETTING) + e.addMessage("for storage " + storage_def.engine->name); + throw; + } + } + else + { + auto settings_ast = std::make_shared(); + settings_ast->is_standalone = false; + storage_def.set(storage_def.settings, settings_ast); + } +} +} diff --git a/src/Storages/NATS/NATSSettings.h b/src/Storages/NATS/NATSSettings.h new file mode 100644 index 00000000000..6029aaea9f6 --- /dev/null +++ b/src/Storages/NATS/NATSSettings.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +namespace DB +{ +class ASTStorage; + +#define NATS_RELATED_SETTINGS(M) \ + M(String, nats_url, "", "A host-port to connect to NATS server.", 0) \ + M(String, nats_subjects, "", "List of subject for NATS table to subscribe/publish to.", 0) \ + M(String, nats_format, "", "The message format.", 0) \ + M(Char, nats_row_delimiter, '\0', "The character to be considered as a delimiter.", 0) \ + M(String, nats_schema, "", "Schema identifier (used by schema-based formats) for NATS engine", 0) \ + M(UInt64, nats_num_consumers, 1, "The number of consumer channels per table.", 0) \ + M(String, nats_queue_group, "", "Name for queue group of NATS subscribers.", 0) \ + M(Bool, nats_secure, false, "Use SSL connection", 0) \ + M(UInt64, nats_max_reconnect, 5, "Maximum amount of reconnection attempts.", 0) \ + M(UInt64, nats_reconnect_wait, 2000, "Amount of time in milliseconds to sleep between each reconnect attempt.", 0) \ + M(String, nats_server_list, "", "Server list for connection", 0) \ + M(UInt64, nats_skip_broken_messages, 0, "Skip at least this number of broken messages from NATS per block", 0) \ + M(UInt64, nats_max_block_size, 0, "Number of row collected before flushing data from NATS.", 0) \ + M(Milliseconds, nats_flush_interval_ms, 0, "Timeout for flushing data from NATS.", 0) \ + M(String, nats_username, "", "NATS username", 0) \ + M(String, nats_password, "", "NATS password", 0) \ + M(String, nats_token, "", "NATS token", 0) + +#define LIST_OF_NATS_SETTINGS(M) \ + NATS_RELATED_SETTINGS(M) \ + FORMAT_FACTORY_SETTINGS(M) + +DECLARE_SETTINGS_TRAITS(NATSSettingsTraits, LIST_OF_NATS_SETTINGS) + +struct NATSSettings : public BaseSettings +{ + void loadFromQuery(ASTStorage & storage_def); +}; +} diff --git a/src/Storages/NATS/NATSSink.cpp b/src/Storages/NATS/NATSSink.cpp new file mode 100644 index 00000000000..44cf51072e6 --- /dev/null +++ b/src/Storages/NATS/NATSSink.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NATSSink::NATSSink( + StorageNATS & storage_, + const StorageMetadataPtr & metadata_snapshot_, + ContextPtr context_, + ProducerBufferPtr buffer_) + : SinkToStorage(metadata_snapshot_->getSampleBlockNonMaterialized()) + , storage(storage_) + , metadata_snapshot(metadata_snapshot_) + , context(context_) + , buffer(buffer_) +{ +} + + +void NATSSink::onStart() +{ + buffer->activateWriting(); + + auto format_settings = getFormatSettings(context); + format_settings.protobuf.allow_multiple_rows_without_delimiter = true; + + format = FormatFactory::instance().getOutputFormat(storage.getFormatName(), *buffer, getHeader(), context, + [this](const Columns & /* columns */, size_t /* rows */) + { + buffer->countRow(); + }, + format_settings); +} + + +void NATSSink::consume(Chunk chunk) +{ + format->write(getHeader().cloneWithColumns(chunk.detachColumns())); +} + + +void NATSSink::onFinish() +{ + format->finalize(); + + if (buffer) + buffer->updateMaxWait(); +} + +} diff --git a/src/Storages/NATS/NATSSink.h b/src/Storages/NATS/NATSSink.h new file mode 100644 index 00000000000..d94575de0e7 --- /dev/null +++ b/src/Storages/NATS/NATSSink.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class IOutputFormat; +using IOutputFormatPtr = std::shared_ptr; + +class NATSSink : public SinkToStorage +{ +public: + explicit NATSSink(StorageNATS & storage_, const StorageMetadataPtr & metadata_snapshot_, ContextPtr context_, ProducerBufferPtr buffer_); + + void onStart() override; + void consume(Chunk chunk) override; + void onFinish() override; + + String getName() const override { return "NATSSink"; } + +private: + StorageNATS & storage; + StorageMetadataPtr metadata_snapshot; + ContextPtr context; + ProducerBufferPtr buffer; + IOutputFormatPtr format; +}; +} diff --git a/src/Storages/NATS/NATSSource.cpp b/src/Storages/NATS/NATSSource.cpp new file mode 100644 index 00000000000..f5e5e4f8b91 --- /dev/null +++ b/src/Storages/NATS/NATSSource.cpp @@ -0,0 +1,135 @@ +#include + +#include +#include +#include +#include + +namespace DB +{ + +static std::pair getHeaders(const StorageSnapshotPtr & storage_snapshot) +{ + auto non_virtual_header = storage_snapshot->metadata->getSampleBlockNonMaterialized(); + auto virtual_header = storage_snapshot->getSampleBlockForColumns({"_subject"}); + + return {non_virtual_header, virtual_header}; +} + +static Block getSampleBlock(const Block & non_virtual_header, const Block & virtual_header) +{ + auto header = non_virtual_header; + for (const auto & column : virtual_header) + header.insert(column); + + return header; +} + +NATSSource::NATSSource( + StorageNATS & storage_, + const StorageSnapshotPtr & storage_snapshot_, + ContextPtr context_, + const Names & columns, + size_t max_block_size_) + : NATSSource(storage_, storage_snapshot_, getHeaders(storage_snapshot_), context_, columns, max_block_size_) +{ +} + +NATSSource::NATSSource( + StorageNATS & storage_, + const StorageSnapshotPtr & storage_snapshot_, + std::pair headers, + ContextPtr context_, + const Names & columns, + size_t max_block_size_) + : ISource(getSampleBlock(headers.first, headers.second)) + , storage(storage_) + , storage_snapshot(storage_snapshot_) + , context(context_) + , column_names(columns) + , max_block_size(max_block_size_) + , non_virtual_header(std::move(headers.first)) + , virtual_header(std::move(headers.second)) +{ + storage.incrementReader(); +} + + +NATSSource::~NATSSource() +{ + storage.decrementReader(); + + if (!buffer) + return; + + buffer->allowNext(); + storage.pushReadBuffer(buffer); +} + +bool NATSSource::checkTimeLimit() const +{ + if (max_execution_time != 0) + { + auto elapsed_ns = total_stopwatch.elapsed(); + + if (elapsed_ns > static_cast(max_execution_time.totalMicroseconds()) * 1000) + return false; + } + + return true; +} + +Chunk NATSSource::generate() +{ + if (!buffer) + { + auto timeout = std::chrono::milliseconds(context->getSettingsRef().rabbitmq_max_wait_ms.totalMilliseconds()); + buffer = storage.popReadBuffer(timeout); + buffer->subscribe(); + } + + if (!buffer || is_finished) + return {}; + + is_finished = true; + + MutableColumns virtual_columns = virtual_header.cloneEmptyColumns(); + auto input_format + = FormatFactory::instance().getInputFormat(storage.getFormatName(), *buffer, non_virtual_header, context, max_block_size); + + StreamingFormatExecutor executor(non_virtual_header, input_format); + + size_t total_rows = 0; + + while (true) + { + if (buffer->eof()) + break; + + auto new_rows = executor.execute(); + + if (new_rows) + { + auto subject = buffer->getSubject(); + virtual_columns[0]->insertMany(subject, new_rows); + + total_rows = total_rows + new_rows; + } + + buffer->allowNext(); + + if (total_rows >= max_block_size || buffer->queueEmpty() || buffer->isConsumerStopped() || !checkTimeLimit()) + break; + } + + if (total_rows == 0) + return {}; + + auto result_columns = executor.getResultColumns(); + for (auto & column : virtual_columns) + result_columns.push_back(std::move(column)); + + return Chunk(std::move(result_columns), total_rows); +} + +} diff --git a/src/Storages/NATS/NATSSource.h b/src/Storages/NATS/NATSSource.h new file mode 100644 index 00000000000..e4e94d2347a --- /dev/null +++ b/src/Storages/NATS/NATSSource.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class NATSSource : public ISource +{ +public: + NATSSource( + StorageNATS & storage_, + const StorageSnapshotPtr & storage_snapshot_, + ContextPtr context_, + const Names & columns, + size_t max_block_size_); + + ~NATSSource() override; + + String getName() const override { return storage.getName(); } + ConsumerBufferPtr getBuffer() { return buffer; } + + Chunk generate() override; + + bool queueEmpty() const { return !buffer || buffer->queueEmpty(); } + + void setTimeLimit(Poco::Timespan max_execution_time_) { max_execution_time = max_execution_time_; } + +private: + bool checkTimeLimit() const; + + StorageNATS & storage; + StorageSnapshotPtr storage_snapshot; + ContextPtr context; + Names column_names; + const size_t max_block_size; + + bool is_finished = false; + const Block non_virtual_header; + const Block virtual_header; + + ConsumerBufferPtr buffer; + + Poco::Timespan max_execution_time = 0; + Stopwatch total_stopwatch {CLOCK_MONOTONIC_COARSE}; + + NATSSource( + StorageNATS & storage_, + const StorageSnapshotPtr & storage_snapshot_, + std::pair headers, + ContextPtr context_, + const Names & columns, + size_t max_block_size_); +}; + +} diff --git a/src/Storages/NATS/ReadBufferFromNATSConsumer.cpp b/src/Storages/NATS/ReadBufferFromNATSConsumer.cpp new file mode 100644 index 00000000000..fa6e60ac213 --- /dev/null +++ b/src/Storages/NATS/ReadBufferFromNATSConsumer.cpp @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include +#include +#include "Poco/Timer.h" +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int CANNOT_CONNECT_NATS; +} + +ReadBufferFromNATSConsumer::ReadBufferFromNATSConsumer( + std::shared_ptr connection_, + StorageNATS & storage_, + std::vector & subjects_, + const String & subscribe_queue_name, + Poco::Logger * log_, + char row_delimiter_, + uint32_t queue_size_, + const std::atomic & stopped_) + : ReadBuffer(nullptr, 0) + , connection(connection_) + , storage(storage_) + , subjects(subjects_) + , log(log_) + , row_delimiter(row_delimiter_) + , stopped(stopped_) + , queue_name(subscribe_queue_name) + , received(queue_size_) +{ +} + +void ReadBufferFromNATSConsumer::subscribe() +{ + if (subscribed) + return; + + for (const auto & subject : subjects) + { + natsSubscription * ns; + auto status = natsConnection_QueueSubscribe( + &ns, connection->getConnection(), subject.c_str(), queue_name.c_str(), onMsg, static_cast(this)); + if (status == NATS_OK) + { + LOG_DEBUG(log, "Subscribed to subject {}", subject); + natsSubscription_SetPendingLimits(ns, -1, -1); + subscriptions.emplace_back(ns, &natsSubscription_Destroy); + } + else + { + throw Exception(ErrorCodes::CANNOT_CONNECT_NATS, "Failed to subscribe to subject {}", subject); + } + } + subscribed = true; +} + +void ReadBufferFromNATSConsumer::unsubscribe() +{ + for (const auto & subscription : subscriptions) + natsSubscription_Unsubscribe(subscription.get()); +} + +bool ReadBufferFromNATSConsumer::nextImpl() +{ + if (stopped || !allowed) + return false; + + if (received.tryPop(current)) + { + auto * new_position = const_cast(current.message.data()); + BufferBase::set(new_position, current.message.size(), 0); + allowed = false; + + return true; + } + + return false; +} + +void ReadBufferFromNATSConsumer::onMsg(natsConnection *, natsSubscription *, natsMsg * msg, void * consumer) +{ + auto * buffer = static_cast(consumer); + const int msg_length = natsMsg_GetDataLength(msg); + + if (msg_length) + { + String message_received = std::string(natsMsg_GetData(msg), msg_length); + String subject = natsMsg_GetSubject(msg); + if (buffer->row_delimiter != '\0') + message_received += buffer->row_delimiter; + + MessageData data = { + .message = message_received, + .subject = subject, + }; + if (!buffer->received.push(std::move(data))) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to received queue"); + + buffer->storage.startStreaming(); + } + + natsMsg_Destroy(msg); +} + +} diff --git a/src/Storages/NATS/ReadBufferFromNATSConsumer.h b/src/Storages/NATS/ReadBufferFromNATSConsumer.h new file mode 100644 index 00000000000..306c0aff3bf --- /dev/null +++ b/src/Storages/NATS/ReadBufferFromNATSConsumer.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace Poco +{ +class Logger; +} + +namespace DB +{ + +class ReadBufferFromNATSConsumer : public ReadBuffer +{ +public: + ReadBufferFromNATSConsumer( + std::shared_ptr connection_, + StorageNATS & storage_, + std::vector & subjects_, + const String & subscribe_queue_name, + Poco::Logger * log_, + char row_delimiter_, + uint32_t queue_size_, + const std::atomic & stopped_); + + struct MessageData + { + String message; + String subject; + }; + + void subscribe(); + void unsubscribe(); + + size_t subjectsCount() { return subjects.size(); } + + bool isConsumerStopped() { return stopped; } + + bool queueEmpty() { return received.empty(); } + size_t queueSize() { return received.size(); } + void allowNext() { allowed = true; } // Allow to read next message. + + auto getSubject() const { return current.subject; } + +private: + bool nextImpl() override; + + static void onMsg(natsConnection * nc, natsSubscription * sub, natsMsg * msg, void * consumer); + + std::shared_ptr connection; + StorageNATS & storage; + std::vector subscriptions; + std::vector subjects; + Poco::Logger * log; + char row_delimiter; + bool allowed = true; + const std::atomic & stopped; + + bool subscribed = false; + String queue_name; + + String channel_id; + ConcurrentBoundedQueue received; + MessageData current; +}; + +} diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp new file mode 100644 index 00000000000..3c1a04c7824 --- /dev/null +++ b/src/Storages/NATS/StorageNATS.cpp @@ -0,0 +1,730 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +static const uint32_t QUEUE_SIZE = 100000; +static const auto RESCHEDULE_MS = 500; +static const auto MAX_THREAD_WORK_DURATION_MS = 60000; + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int CANNOT_CONNECT_NATS; + extern const int QUERY_NOT_ALLOWED; +} + + +StorageNATS::StorageNATS( + const StorageID & table_id_, + ContextPtr context_, + const ColumnsDescription & columns_, + std::unique_ptr nats_settings_, + bool is_attach_) + : IStorage(table_id_) + , WithContext(context_->getGlobalContext()) + , nats_settings(std::move(nats_settings_)) + , subjects(parseList(getContext()->getMacros()->expand(nats_settings->nats_subjects), ',')) + , format_name(getContext()->getMacros()->expand(nats_settings->nats_format)) + , row_delimiter(nats_settings->nats_row_delimiter.value) + , schema_name(getContext()->getMacros()->expand(nats_settings->nats_schema)) + , num_consumers(nats_settings->nats_num_consumers.value) + , log(&Poco::Logger::get("StorageNATS (" + table_id_.table_name + ")")) + , semaphore(0, num_consumers) + , queue_size(std::max(QUEUE_SIZE, static_cast(getMaxBlockSize()))) + , is_attach(is_attach_) +{ + auto nats_username = getContext()->getMacros()->expand(nats_settings->nats_username); + auto nats_password = getContext()->getMacros()->expand(nats_settings->nats_password); + auto nats_token = getContext()->getMacros()->expand(nats_settings->nats_token); + + configuration = + { + .url = getContext()->getMacros()->expand(nats_settings->nats_url), + .servers = parseList(getContext()->getMacros()->expand(nats_settings->nats_server_list), ','), + .username = nats_username.empty() ? getContext()->getConfigRef().getString("nats.user", "") : nats_username, + .password = nats_password.empty() ? getContext()->getConfigRef().getString("nats.password", "") : nats_password, + .token = nats_token.empty() ? getContext()->getConfigRef().getString("nats.token", "") : nats_token, + .max_reconnect = static_cast(nats_settings->nats_max_reconnect.value), + .reconnect_wait = static_cast(nats_settings->nats_reconnect_wait.value), + .secure = nats_settings->nats_secure.value + }; + + if (configuration.secure) + SSL_library_init(); + + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns_); + setInMemoryMetadata(storage_metadata); + + nats_context = addSettings(getContext()); + nats_context->makeQueryContext(); + + try + { + connection = std::make_shared(configuration, log); + if (!connection->connect()) + throw Exception(ErrorCodes::CANNOT_CONNECT_NATS, "Cannot connect to {}. Nats last error: {}", + connection->connectionInfoForLog(), nats_GetLastError(nullptr)); + } + catch (...) + { + tryLogCurrentException(log); + if (!is_attach) + throw; + } + + /// One looping task for all consumers as they share the same connection == the same handler == the same event loop + looping_task = getContext()->getMessageBrokerSchedulePool().createTask("NATSLoopingTask", [this] { loopingFunc(); }); + looping_task->deactivate(); + + streaming_task = getContext()->getMessageBrokerSchedulePool().createTask("NATSStreamingTask", [this] { streamingToViewsFunc(); }); + streaming_task->deactivate(); + + connection_task = getContext()->getMessageBrokerSchedulePool().createTask("NATSConnectionManagerTask", [this] { connectionFunc(); }); + connection_task->deactivate(); +} + + +Names StorageNATS::parseList(const String & list, char delim) +{ + Names result; + if (list.empty()) + return result; + boost::split(result, list, [delim](char c) { return c == delim; }); + for (String & key : result) + boost::trim(key); + + return result; +} + + +String StorageNATS::getTableBasedName(String name, const StorageID & table_id) +{ + if (name.empty()) + return fmt::format("{}_{}", table_id.database_name, table_id.table_name); + else + return fmt::format("{}_{}_{}", name, table_id.database_name, table_id.table_name); +} + + +ContextMutablePtr StorageNATS::addSettings(ContextPtr local_context) const +{ + auto modified_context = Context::createCopy(local_context); + modified_context->setSetting("input_format_skip_unknown_fields", true); + modified_context->setSetting("input_format_allow_errors_ratio", 0.); + modified_context->setSetting("input_format_allow_errors_num", nats_settings->nats_skip_broken_messages.value); + + if (!schema_name.empty()) + modified_context->setSetting("format_schema", schema_name); + + for (const auto & setting : *nats_settings) + { + const auto & setting_name = setting.getName(); + + /// check for non-nats-related settings + if (!setting_name.starts_with("nats_")) + modified_context->setSetting(setting_name, setting.getValue()); + } + + return modified_context; +} + + +void StorageNATS::loopingFunc() +{ + connection->getHandler().startLoop(); + looping_task->activateAndSchedule(); +} + + +void StorageNATS::stopLoop() +{ + connection->getHandler().updateLoopState(Loop::STOP); +} + +void StorageNATS::stopLoopIfNoReaders() +{ + /// Stop the loop if no select was started. + /// There can be a case that selects are finished + /// but not all sources decremented the counter, then + /// it is ok that the loop is not stopped, because + /// there is a background task (streaming_task), which + /// also checks whether there is an idle loop. + std::lock_guard lock(loop_mutex); + if (readers_count) + return; + connection->getHandler().updateLoopState(Loop::STOP); +} + +void StorageNATS::startLoop() +{ + connection->getHandler().updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); +} + + +void StorageNATS::incrementReader() +{ + ++readers_count; +} + + +void StorageNATS::decrementReader() +{ + --readers_count; +} + + +void StorageNATS::connectionFunc() +{ + if (consumers_ready) + return; + + bool needs_rescheduling = true; + if (connection->reconnect()) + needs_rescheduling &= !initBuffers(); + + if (needs_rescheduling) + connection_task->scheduleAfter(RESCHEDULE_MS); +} + +bool StorageNATS::initBuffers() +{ + size_t num_initialized = 0; + for (auto & buffer : buffers) + { + try + { + buffer->subscribe(); + ++num_initialized; + } + catch (...) + { + tryLogCurrentException(log); + break; + } + } + + startLoop(); + const bool are_buffers_initialized = num_initialized == num_created_consumers; + if (are_buffers_initialized) + consumers_ready.store(true); + return are_buffers_initialized; +} + + +/* Need to deactivate this way because otherwise might get a deadlock when first deactivate streaming task in shutdown and then + * inside streaming task try to deactivate any other task + */ +void StorageNATS::deactivateTask(BackgroundSchedulePool::TaskHolder & task, bool stop_loop) +{ + if (stop_loop) + stopLoop(); + + std::unique_lock lock(task_mutex, std::defer_lock); + lock.lock(); + task->deactivate(); +} + + +size_t StorageNATS::getMaxBlockSize() const +{ + return nats_settings->nats_max_block_size.changed ? nats_settings->nats_max_block_size.value + : (getContext()->getSettingsRef().max_insert_block_size.value / num_consumers); +} + + +void StorageNATS::read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr local_context, + QueryProcessingStage::Enum /* processed_stage */, + size_t /* max_block_size */, + unsigned /* num_streams */) +{ + if (!consumers_ready) + throw Exception("NATS consumers setup not finished. Connection might be lost", ErrorCodes::CANNOT_CONNECT_NATS); + + if (num_created_consumers == 0) + return; + + if (!local_context->getSettingsRef().stream_like_engine_allow_direct_select) + throw Exception( + ErrorCodes::QUERY_NOT_ALLOWED, "Direct select is not allowed. To enable use setting `stream_like_engine_allow_direct_select`"); + + if (mv_attached) + throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "Cannot read from StorageNATS with attached materialized views"); + + std::lock_guard lock(loop_mutex); + + auto sample_block = storage_snapshot->getSampleBlockForColumns(column_names); + auto modified_context = addSettings(local_context); + + if (!connection->isConnected()) + { + if (!connection->reconnect()) + throw Exception(ErrorCodes::CANNOT_CONNECT_NATS, "No connection to {}", connection->connectionInfoForLog()); + } + + Pipes pipes; + pipes.reserve(num_created_consumers); + + for (size_t i = 0; i < num_created_consumers; ++i) + { + auto nats_source = std::make_shared(*this, storage_snapshot, modified_context, column_names, 1); + + auto converting_dag = ActionsDAG::makeConvertingActions( + nats_source->getPort().getHeader().getColumnsWithTypeAndName(), + sample_block.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + + auto converting = std::make_shared(std::move(converting_dag)); + auto converting_transform = std::make_shared(nats_source->getPort().getHeader(), std::move(converting)); + + pipes.emplace_back(std::move(nats_source)); + pipes.back().addTransform(std::move(converting_transform)); + } + + if (!connection->getHandler().loopRunning() && connection->isConnected()) + startLoop(); + + LOG_DEBUG(log, "Starting reading {} streams", pipes.size()); + auto pipe = Pipe::unitePipes(std::move(pipes)); + + if (pipe.empty()) + { + auto header = storage_snapshot->getSampleBlockForColumns(column_names); + InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info, local_context); + } + else + { + auto read_step = std::make_unique(std::move(pipe), getName(), query_info.storage_limits); + query_plan.addStep(std::move(read_step)); + query_plan.addInterpreterContext(modified_context); + } +} + + +SinkToStoragePtr StorageNATS::write(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context) +{ + auto modified_context = addSettings(local_context); + std::string subject = modified_context->getSettingsRef().stream_like_engine_insert_queue.changed + ? modified_context->getSettingsRef().stream_like_engine_insert_queue.value + : ""; + if (subject.empty()) + { + if (subjects.size() > 1) + { + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "This NATS engine reads from multiple subjects. You must specify `stream_like_engine_insert_queue` to choose the subject to write to"); + } + else + { + subject = subjects[0]; + } + } + + auto pos = subject.find('*'); + if (pos != std::string::npos || subject.back() == '>') + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Can not publish to wildcard subject"); + + if (!isSubjectInSubscriptions(subject)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Selected subject is not among engine subjects"); + + return std::make_shared(*this, metadata_snapshot, local_context, createWriteBuffer(subject)); +} + + +void StorageNATS::startup() +{ + for (size_t i = 0; i < num_consumers; ++i) + { + try + { + auto buffer = createReadBuffer(); + pushReadBuffer(std::move(buffer)); + ++num_created_consumers; + } + catch (...) + { + if (!is_attach) + throw; + tryLogCurrentException(log); + } + } + + if (!connection->isConnected() || !initBuffers()) + connection_task->activateAndSchedule(); +} + + +void StorageNATS::shutdown() +{ + shutdown_called = true; + + /// In case it has not yet been able to setup connection; + deactivateTask(connection_task, false); + + /// The order of deactivating tasks is important: wait for streamingToViews() func to finish and + /// then wait for background event loop to finish. + deactivateTask(streaming_task, false); + deactivateTask(looping_task, true); + + /// Just a paranoid try catch, it is not actually needed. + try + { + if (drop_table) + { + for (auto & buffer : buffers) + buffer->unsubscribe(); + } + + connection->disconnect(); + + for (size_t i = 0; i < num_created_consumers; ++i) + popReadBuffer(); + } + catch (...) + { + tryLogCurrentException(log); + } +} + +void StorageNATS::pushReadBuffer(ConsumerBufferPtr buffer) +{ + std::lock_guard lock(buffers_mutex); + buffers.push_back(buffer); + semaphore.set(); +} + + +ConsumerBufferPtr StorageNATS::popReadBuffer() +{ + return popReadBuffer(std::chrono::milliseconds::zero()); +} + + +ConsumerBufferPtr StorageNATS::popReadBuffer(std::chrono::milliseconds timeout) +{ + // Wait for the first free buffer + if (timeout == std::chrono::milliseconds::zero()) + semaphore.wait(); + else + { + if (!semaphore.tryWait(timeout.count())) + return nullptr; + } + + // Take the first available buffer from the list + std::lock_guard lock(buffers_mutex); + auto buffer = buffers.back(); + buffers.pop_back(); + + return buffer; +} + + +ConsumerBufferPtr StorageNATS::createReadBuffer() +{ + return std::make_shared( + connection, *this, subjects, + nats_settings->nats_queue_group.changed ? nats_settings->nats_queue_group.value : getStorageID().getFullTableName(), + log, row_delimiter, queue_size, shutdown_called); +} + + +ProducerBufferPtr StorageNATS::createWriteBuffer(const std::string & subject) +{ + return std::make_shared( + configuration, getContext(), subject, shutdown_called, log, + row_delimiter ? std::optional{row_delimiter} : std::nullopt, 1, 1024); +} + +bool StorageNATS::isSubjectInSubscriptions(const std::string & subject) +{ + auto subject_levels = parseList(subject, '.'); + + for (const auto & nats_subject : subjects) + { + auto nats_subject_levels = parseList(nats_subject, '.'); + size_t levels_to_check = 0; + if (!nats_subject_levels.empty() && nats_subject_levels.back() == ">") + levels_to_check = nats_subject_levels.size() - 1; + if (levels_to_check) + { + if (subject_levels.size() < levels_to_check) + continue; + } + else + { + if (subject_levels.size() != nats_subject_levels.size()) + continue; + levels_to_check = nats_subject_levels.size(); + } + + bool is_same = true; + for (size_t i = 0; i < levels_to_check; ++i) + { + if (nats_subject_levels[i] == "*") + continue; + + if (subject_levels[i] != nats_subject_levels[i]) + { + is_same = false; + break; + } + } + if (is_same) + return true; + } + + return false; +} + + +bool StorageNATS::checkDependencies(const StorageID & table_id) +{ + // Check if all dependencies are attached + auto dependencies = DatabaseCatalog::instance().getDependencies(table_id); + if (dependencies.empty()) + return true; + + // Check the dependencies are ready? + for (const auto & db_tab : dependencies) + { + auto table = DatabaseCatalog::instance().tryGetTable(db_tab, getContext()); + if (!table) + return false; + + // If it materialized view, check it's target table + auto * materialized_view = dynamic_cast(table.get()); + if (materialized_view && !materialized_view->tryGetTargetTable()) + return false; + + // Check all its dependencies + if (!checkDependencies(db_tab)) + return false; + } + + return true; +} + + +void StorageNATS::streamingToViewsFunc() +{ + bool do_reschedule = true; + try + { + auto table_id = getStorageID(); + + // Check if at least one direct dependency is attached + size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); + bool nats_connected = connection->isConnected() || connection->reconnect(); + + if (dependencies_count && nats_connected) + { + auto start_time = std::chrono::steady_clock::now(); + + mv_attached.store(true); + + // Keep streaming as long as there are attached views and streaming is not cancelled + while (!shutdown_called && num_created_consumers > 0) + { + if (!checkDependencies(table_id)) + break; + + LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); + + if (streamToViews()) + { + /// Reschedule with backoff. + do_reschedule = false; + break; + } + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time); + if (duration.count() > MAX_THREAD_WORK_DURATION_MS) + { + LOG_TRACE(log, "Reschedule streaming. Thread work duration limit exceeded."); + break; + } + } + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + + mv_attached.store(false); + + if (!shutdown_called && do_reschedule) + streaming_task->scheduleAfter(RESCHEDULE_MS); +} + + +bool StorageNATS::streamToViews() +{ + auto table_id = getStorageID(); + auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); + if (!table) + throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); + + // Create an INSERT query for streaming data + auto insert = std::make_shared(); + insert->table_id = table_id; + + // Only insert into dependent views and expect that input blocks contain virtual columns + InterpreterInsertQuery interpreter(insert, nats_context, false, true, true); + auto block_io = interpreter.execute(); + + auto storage_snapshot = getStorageSnapshot(getInMemoryMetadataPtr(), getContext()); + auto column_names = block_io.pipeline.getHeader().getNames(); + auto sample_block = storage_snapshot->getSampleBlockForColumns(column_names); + + auto block_size = getMaxBlockSize(); + + // Create a stream for each consumer and join them in a union stream + std::vector> sources; + Pipes pipes; + sources.reserve(num_created_consumers); + pipes.reserve(num_created_consumers); + + for (size_t i = 0; i < num_created_consumers; ++i) + { + LOG_DEBUG(log, "Current queue size: {}", buffers[0]->queueSize()); + auto source = std::make_shared(*this, storage_snapshot, nats_context, column_names, block_size); + sources.emplace_back(source); + pipes.emplace_back(source); + + Poco::Timespan max_execution_time = nats_settings->nats_flush_interval_ms.changed + ? nats_settings->nats_flush_interval_ms + : getContext()->getSettingsRef().stream_flush_interval_ms; + + source->setTimeLimit(max_execution_time); + } + + block_io.pipeline.complete(Pipe::unitePipes(std::move(pipes))); + + if (!connection->getHandler().loopRunning()) + startLoop(); + + { + CompletedPipelineExecutor executor(block_io.pipeline); + executor.execute(); + } + + size_t queue_empty = 0; + + if (!connection->isConnected()) + { + if (shutdown_called) + return true; + + if (connection->reconnect()) + { + LOG_DEBUG(log, "Connection restored"); + } + else + { + LOG_TRACE(log, "Reschedule streaming. Unable to restore connection."); + return true; + } + } + else + { + for (auto & source : sources) + { + if (source->queueEmpty()) + ++queue_empty; + + connection->getHandler().iterateLoop(); + } + } + + if (queue_empty == num_created_consumers) + { + LOG_TRACE(log, "Reschedule streaming. Queues are empty."); + return true; + } + else + { + startLoop(); + } + + /// Do not reschedule, do not stop event loop. + return false; +} + + +void registerStorageNATS(StorageFactory & factory) +{ + auto creator_fn = [](const StorageFactory::Arguments & args) + { + auto nats_settings = std::make_unique(); + bool with_named_collection = getExternalDataSourceConfiguration(args.engine_args, *nats_settings, args.getLocalContext()); + if (!with_named_collection && !args.storage_def->settings) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "NATS engine must have settings"); + + nats_settings->loadFromQuery(*args.storage_def); + + if (!nats_settings->nats_url.changed && !nats_settings->nats_server_list.changed) + throw Exception( + "You must specify either `nats_url` or `nats_server_list` settings", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (!nats_settings->nats_format.changed) + throw Exception("You must specify `nats_format` setting", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (!nats_settings->nats_subjects.changed) + throw Exception("You must specify `nats_subjects` setting", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(args.table_id, args.getContext(), args.columns, std::move(nats_settings), args.attach); + }; + + factory.registerStorage("NATS", creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, }); +} + + +NamesAndTypesList StorageNATS::getVirtuals() const +{ + return NamesAndTypesList{ + {"_subject", std::make_shared()} + }; +} + +} diff --git a/src/Storages/NATS/StorageNATS.h b/src/Storages/NATS/StorageNATS.h new file mode 100644 index 00000000000..185b39250c8 --- /dev/null +++ b/src/Storages/NATS/StorageNATS.h @@ -0,0 +1,145 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class StorageNATS final : public IStorage, WithContext +{ +public: + StorageNATS( + const StorageID & table_id_, + ContextPtr context_, + const ColumnsDescription & columns_, + std::unique_ptr nats_settings_, + bool is_attach_); + + std::string getName() const override { return "NATS"; } + + bool noPushingToViews() const override { return true; } + + void startup() override; + void shutdown() override; + + /// This is a bad way to let storage know in shutdown() that table is going to be dropped. There are some actions which need + /// to be done only when table is dropped (not when detached). Also connection must be closed only in shutdown, but those + /// actions require an open connection. Therefore there needs to be a way inside shutdown() method to know whether it is called + /// because of drop query. And drop() method is not suitable at all, because it will not only require to reopen connection, but also + /// it can be called considerable time after table is dropped (for example, in case of Atomic database), which is not appropriate for the case. + void checkTableCanBeDropped() const override { drop_table = true; } + + /// Always return virtual columns in addition to required columns + void read( + QueryPlan & query_plan, + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr local_context, + QueryProcessingStage::Enum /* processed_stage */, + size_t /* max_block_size */, + unsigned /* num_streams */) override; + + SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override; + + void pushReadBuffer(ConsumerBufferPtr buf); + ConsumerBufferPtr popReadBuffer(); + ConsumerBufferPtr popReadBuffer(std::chrono::milliseconds timeout); + + const String & getFormatName() const { return format_name; } + NamesAndTypesList getVirtuals() const override; + + void incrementReader(); + void decrementReader(); + + void startStreaming() { if (!mv_attached) { streaming_task->activateAndSchedule(); } } + +private: + ContextMutablePtr nats_context; + std::unique_ptr nats_settings; + std::vector subjects; + + const String format_name; + char row_delimiter; + const String schema_name; + size_t num_consumers; + + Poco::Logger * log; + + NATSConnectionManagerPtr connection; /// Connection for all consumers + NATSConfiguration configuration; + + size_t num_created_consumers = 0; + Poco::Semaphore semaphore; + std::mutex buffers_mutex; + std::vector buffers; /// available buffers for NATS consumers + + /// maximum number of messages in NATS queue (x-max-length). Also used + /// to setup size of inner buffer for received messages + uint32_t queue_size; + + std::once_flag flag; /// remove exchange only once + std::mutex task_mutex; + BackgroundSchedulePool::TaskHolder streaming_task; + BackgroundSchedulePool::TaskHolder looping_task; + BackgroundSchedulePool::TaskHolder connection_task; + + /// True if consumers have subscribed to all subjects + std::atomic consumers_ready{false}; + /// Needed for tell MV or producer background tasks + /// that they must finish as soon as possible. + std::atomic shutdown_called{false}; + /// For select query we must be aware of the end of streaming + /// to be able to turn off the loop. + std::atomic readers_count = 0; + std::atomic mv_attached = false; + + /// In select query we start event loop, but do not stop it + /// after that select is finished. Then in a thread, which + /// checks for MV we also check if we have select readers. + /// If not - we turn off the loop. The checks are done under + /// mutex to avoid having a turned off loop when select was + /// started. + std::mutex loop_mutex; + + mutable bool drop_table = false; + bool is_attach; + + ConsumerBufferPtr createReadBuffer(); + ProducerBufferPtr createWriteBuffer(const std::string & subject); + + bool isSubjectInSubscriptions(const std::string & subject); + + + /// Functions working in the background + void streamingToViewsFunc(); + void loopingFunc(); + void connectionFunc(); + + bool initBuffers(); + + void startLoop(); + void stopLoop(); + void stopLoopIfNoReaders(); + + static Names parseList(const String & list, char delim); + static String getTableBasedName(String name, const StorageID & table_id); + + ContextMutablePtr addSettings(ContextPtr context) const; + size_t getMaxBlockSize() const; + void deactivateTask(BackgroundSchedulePool::TaskHolder & task, bool stop_loop); + + bool streamToViews(); + bool checkDependencies(const StorageID & table_id); +}; + +} diff --git a/src/Storages/NATS/WriteBufferToNATSProducer.cpp b/src/Storages/NATS/WriteBufferToNATSProducer.cpp new file mode 100644 index 00000000000..af76247d903 --- /dev/null +++ b/src/Storages/NATS/WriteBufferToNATSProducer.cpp @@ -0,0 +1,183 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +static const auto BATCH = 1000; +static const auto MAX_BUFFERED = 131072; + +namespace ErrorCodes +{ + extern const int CANNOT_CONNECT_NATS; + extern const int LOGICAL_ERROR; +} + +WriteBufferToNATSProducer::WriteBufferToNATSProducer( + const NATSConfiguration & configuration_, + ContextPtr global_context, + const String & subject_, + std::atomic & shutdown_called_, + Poco::Logger * log_, + std::optional delimiter, + size_t rows_per_message, + size_t chunk_size_) + : WriteBuffer(nullptr, 0) + , connection(configuration_, log_) + , subject(subject_) + , shutdown_called(shutdown_called_) + , payloads(BATCH) + , log(log_) + , delim(delimiter) + , max_rows(rows_per_message) + , chunk_size(chunk_size_) +{ + if (!connection.connect()) + throw Exception(ErrorCodes::CANNOT_CONNECT_NATS, "Cannot connect to NATS {}", connection.connectionInfoForLog()); + + writing_task = global_context->getSchedulePool().createTask("NATSWritingTask", [this] { writingFunc(); }); + writing_task->deactivate(); + + reinitializeChunks(); +} + + +WriteBufferToNATSProducer::~WriteBufferToNATSProducer() +{ + writing_task->deactivate(); + assert(rows == 0); +} + + +void WriteBufferToNATSProducer::countRow() +{ + if (++rows % max_rows == 0) + { + const std::string & last_chunk = chunks.back(); + size_t last_chunk_size = offset(); + + if (last_chunk_size && delim && last_chunk[last_chunk_size - 1] == delim) + --last_chunk_size; + + std::string payload; + payload.reserve((chunks.size() - 1) * chunk_size + last_chunk_size); + + for (auto i = chunks.begin(), end = --chunks.end(); i != end; ++i) + payload.append(*i); + + payload.append(last_chunk, 0, last_chunk_size); + + reinitializeChunks(); + + if (!payloads.push(payload)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to payloads queue"); + } +} + +void WriteBufferToNATSProducer::publish() +{ + uv_thread_t flush_thread; + + uv_thread_create(&flush_thread, publishThreadFunc, static_cast(this)); + + connection.getHandler().startLoop(); + uv_thread_join(&flush_thread); +} + +void WriteBufferToNATSProducer::publishThreadFunc(void * arg) +{ + WriteBufferToNATSProducer * buffer = static_cast(arg); + String payload; + + natsStatus status; + while (!buffer->payloads.empty()) + { + if (natsConnection_Buffered(buffer->connection.getConnection()) > MAX_BUFFERED) + break; + bool pop_result = buffer->payloads.pop(payload); + + if (!pop_result) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not pop payload"); + status = natsConnection_PublishString(buffer->connection.getConnection(), buffer->subject.c_str(), payload.c_str()); + + if (status != NATS_OK) + { + LOG_DEBUG(buffer->log, "Something went wrong during publishing to NATS subject. Nats status text: {}. Last error message: {}", + natsStatus_GetText(status), nats_GetLastError(nullptr)); + if (!buffer->payloads.push(std::move(payload))) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to payloads queue"); + break; + } + } + + nats_ReleaseThreadMemory(); +} + + +void WriteBufferToNATSProducer::writingFunc() +{ + try + { + while ((!payloads.empty() || wait_all) && !shutdown_called.load()) + { + publish(); + + LOG_DEBUG( + log, "Writing func {} {} {}", wait_payloads.load(), payloads.empty(), natsConnection_Buffered(connection.getConnection())); + if (wait_payloads.load() && payloads.empty() && natsConnection_Buffered(connection.getConnection()) == 0) + wait_all = false; + + if (!connection.isConnected() && wait_all) + connection.reconnect(); + + iterateEventLoop(); + } + } + catch (...) + { + tryLogCurrentException(log); + } + + LOG_DEBUG(log, "Producer on subject {} completed", subject); +} + + +void WriteBufferToNATSProducer::nextImpl() +{ + addChunk(); +} + +void WriteBufferToNATSProducer::addChunk() +{ + chunks.push_back(std::string()); + chunks.back().resize(chunk_size); + set(chunks.back().data(), chunk_size); +} + +void WriteBufferToNATSProducer::reinitializeChunks() +{ + rows = 0; + chunks.clear(); + /// We cannot leave the buffer in the undefined state (i.e. without any + /// underlying buffer), since in this case the WriteBuffeR::next() will + /// not call our nextImpl() (due to available() == 0) + addChunk(); +} + + +void WriteBufferToNATSProducer::iterateEventLoop() +{ + connection.getHandler().iterateLoop(); +} + +} diff --git a/src/Storages/NATS/WriteBufferToNATSProducer.h b/src/Storages/NATS/WriteBufferToNATSProducer.h new file mode 100644 index 00000000000..484d80598db --- /dev/null +++ b/src/Storages/NATS/WriteBufferToNATSProducer.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class WriteBufferToNATSProducer : public WriteBuffer +{ +public: + WriteBufferToNATSProducer( + const NATSConfiguration & configuration_, + ContextPtr global_context, + const String & subject_, + std::atomic & shutdown_called_, + Poco::Logger * log_, + std::optional delimiter, + size_t rows_per_message, + size_t chunk_size_); + + ~WriteBufferToNATSProducer() override; + + void countRow(); + void activateWriting() { writing_task->activateAndSchedule(); } + void updateMaxWait() { wait_payloads.store(true); } + +private: + void nextImpl() override; + void addChunk(); + void reinitializeChunks(); + + void iterateEventLoop(); + void writingFunc(); + void publish(); + + static void publishThreadFunc(void * arg); + + NATSConnectionManager connection; + const String subject; + + /* false: when shutdown is called + * true: in all other cases + */ + std::atomic & shutdown_called; + + BackgroundSchedulePool::TaskHolder writing_task; + + /* payloads.queue: + * - payloads are pushed to queue in countRow and popped by another thread in writingFunc, each payload gets into queue only once + */ + ConcurrentBoundedQueue payloads; + + /* false: message delivery successfully ended: publisher received confirm from server that all published + * 1) persistent messages were written to disk + * 2) non-persistent messages reached the queue + * true: continue to process deliveries and returned messages + */ + bool wait_all = true; + + /* false: until writeSuffix is called + * true: means payloads.queue will not grow anymore + */ + std::atomic wait_payloads = false; + + Poco::Logger * log; + const std::optional delim; + const size_t max_rows; + const size_t chunk_size; + size_t rows = 0; + std::list chunks; +}; + +} diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.h b/src/Storages/RabbitMQ/RabbitMQConnection.h index acc3c48f85b..7a355afea0e 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.h +++ b/src/Storages/RabbitMQ/RabbitMQConnection.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 58f08c48c68..73f0c8bd44e 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -93,18 +93,40 @@ StorageRabbitMQ::StorageRabbitMQ( , milliseconds_to_wait(RESCHEDULE_MS) , is_attach(is_attach_) { - auto parsed_address = parseAddress(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_host_port), 5672); - context_->getRemoteHostFilter().checkHostAndPort(parsed_address.first, toString(parsed_address.second)); + const auto & config = getContext()->getConfigRef(); + + std::pair parsed_address; + auto setting_rabbitmq_username = rabbitmq_settings->rabbitmq_username.value; + auto setting_rabbitmq_password = rabbitmq_settings->rabbitmq_password.value; + String username, password; + + if (rabbitmq_settings->rabbitmq_host_port.changed) + { + username = setting_rabbitmq_username.empty() ? config.getString("rabbitmq.username", "") : setting_rabbitmq_username; + password = setting_rabbitmq_password.empty() ? config.getString("rabbitmq.password", "") : setting_rabbitmq_password; + if (username.empty() || password.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "No username or password. They can be specified either in config or in storage settings"); + + parsed_address = parseAddress(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_host_port), 5672); + if (parsed_address.first.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Host or port is incorrect (host: {}, port: {})", parsed_address.first, parsed_address.second); + + context_->getRemoteHostFilter().checkHostAndPort(parsed_address.first, toString(parsed_address.second)); + } + else if (!rabbitmq_settings->rabbitmq_address.changed) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "RabbitMQ requires either `rabbitmq_host_port` or `rabbitmq_address` setting"); - auto rabbitmq_username = rabbitmq_settings->rabbitmq_username.value; - auto rabbitmq_password = rabbitmq_settings->rabbitmq_password.value; configuration = { .host = parsed_address.first, .port = parsed_address.second, - .username = rabbitmq_username.empty() ? getContext()->getConfigRef().getString("rabbitmq.username") : rabbitmq_username, - .password = rabbitmq_password.empty() ? getContext()->getConfigRef().getString("rabbitmq.password") : rabbitmq_password, - .vhost = getContext()->getConfigRef().getString("rabbitmq.vhost", getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_vhost)), + .username = username, + .password = password, + .vhost = config.getString("rabbitmq.vhost", getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_vhost)), .secure = rabbitmq_settings->rabbitmq_secure.value, .connection_string = getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_address) }; @@ -1064,9 +1086,6 @@ bool StorageRabbitMQ::streamToViews() sources.emplace_back(source); pipes.emplace_back(source); - // Limit read batch to maximum block size to allow DDL - StreamLocalLimits limits; - Poco::Timespan max_execution_time = rabbitmq_settings->rabbitmq_flush_interval_ms.changed ? rabbitmq_settings->rabbitmq_flush_interval_ms : getContext()->getSettingsRef().stream_flush_interval_ms; diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 1bfaabe142b..fd943fbe1c5 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -29,12 +30,14 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -47,14 +50,14 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int ROCKSDB_ERROR; + extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ROCKSDB_ERROR; } using FieldVectorPtr = std::shared_ptr; using RocksDBOptions = std::unordered_map; - static RocksDBOptions getOptionsFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & path) { RocksDBOptions options; @@ -71,7 +74,6 @@ static RocksDBOptions getOptionsFromConfig(const Poco::Util::AbstractConfigurati return options; } - // returns keys may be filter by condition static bool traverseASTFilter( const String & primary_key, const DataTypePtr & primary_key_type, const ASTPtr & elem, const PreparedSets & sets, FieldVectorPtr & res) @@ -160,7 +162,6 @@ static bool traverseASTFilter( return false; } - /** Retrieve from the query a condition of the form `key = 'key'`, `key in ('xxx_'), from conjunctions in the WHERE clause. * TODO support key like search */ @@ -176,6 +177,72 @@ static std::pair getFilterKeys( return std::make_pair(res, !matched_keys); } +template +static void fillColumns(const K & key, const V & value, size_t key_pos, const Block & header, MutableColumns & columns) +{ + ReadBufferFromString key_buffer(key); + ReadBufferFromString value_buffer(value); + for (size_t i = 0; i < header.columns(); ++i) + { + const auto & serialization = header.getByPosition(i).type->getDefaultSerialization(); + serialization->deserializeBinary(*columns[i], i == key_pos ? key_buffer : value_buffer); + } +} + +static std::vector serializeKeysToRawString( + FieldVector::const_iterator & it, + FieldVector::const_iterator end, + DataTypePtr key_column_type, + size_t max_block_size) +{ + size_t num_keys = end - it; + + std::vector result; + result.reserve(num_keys); + + size_t rows_processed = 0; + while (it < end && (max_block_size == 0 || rows_processed < max_block_size)) + { + std::string & serialized_key = result.emplace_back(); + WriteBufferFromString wb(serialized_key); + key_column_type->getDefaultSerialization()->serializeBinary(*it, wb); + wb.finalize(); + + ++it; + ++rows_processed; + } + return result; +} + +static std::vector serializeKeysToRawString(const ColumnWithTypeAndName & keys) +{ + if (!keys.column) + return {}; + + size_t num_keys = keys.column->size(); + std::vector result; + result.reserve(num_keys); + + for (size_t i = 0; i < num_keys; ++i) + { + std::string & serialized_key = result.emplace_back(); + WriteBufferFromString wb(serialized_key); + Field field; + keys.column->get(i, field); + /// TODO(@vdimir): use serializeBinaryBulk + keys.type->getDefaultSerialization()->serializeBinary(field, wb); + wb.finalize(); + } + return result; +} + +/// In current implementation rocks db can have key with only one column. +static size_t getPrimaryKeyPos(const Block & header, const Names & primary_key) +{ + if (primary_key.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "RocksDB: only one primary key is supported"); + return header.getPositionByName(primary_key[0]); +} class EmbeddedRocksDBSource : public ISource { @@ -189,7 +256,7 @@ public: const size_t max_block_size_) : ISource(header) , storage(storage_) - , primary_key_pos(header.getPositionByName(storage.getPrimaryKey())) + , primary_key_pos(getPrimaryKeyPos(header, storage.getPrimaryKey())) , keys(keys_) , begin(begin_) , end(end_) @@ -205,7 +272,7 @@ public: const size_t max_block_size_) : ISource(header) , storage(storage_) - , primary_key_pos(header.getPositionByName(storage.getPrimaryKey())) + , primary_key_pos(getPrimaryKeyPos(header, storage.getPrimaryKey())) , iterator(std::move(iterator_)) , max_block_size(max_block_size_) { @@ -222,45 +289,16 @@ public: Chunk generateWithKeys() { - if (it >= end) - return {}; - - size_t num_keys = end - begin; - - std::vector serialized_keys(num_keys); - std::vector slices_keys(num_keys); - const auto & sample_block = getPort().getHeader(); - - const auto & key_column_type = sample_block.getByName(storage.getPrimaryKey()).type; - - size_t rows_processed = 0; - while (it < end && rows_processed < max_block_size) + if (it >= end) { - WriteBufferFromString wb(serialized_keys[rows_processed]); - key_column_type->getDefaultSerialization()->serializeBinary(*it, wb); - wb.finalize(); - slices_keys[rows_processed] = serialized_keys[rows_processed]; - - ++it; - ++rows_processed; + it = {}; + return {}; } - MutableColumns columns = sample_block.cloneEmptyColumns(); - std::vector values; - auto statuses = storage.multiGet(slices_keys, values); - for (size_t i = 0; i < statuses.size(); ++i) - { - if (statuses[i].ok()) - { - ReadBufferFromString key_buffer(slices_keys[i]); - ReadBufferFromString value_buffer(values[i]); - fillColumns(key_buffer, value_buffer, columns); - } - } - - UInt64 num_rows = columns.at(0)->size(); - return Chunk(std::move(columns), num_rows); + const auto & key_column_type = sample_block.getByName(storage.getPrimaryKey().at(0)).type; + auto raw_keys = serializeKeysToRawString(it, end, key_column_type, max_block_size); + return storage.getBySerializedKeys(raw_keys, nullptr); } Chunk generateFullScan() @@ -273,9 +311,7 @@ public: for (size_t rows = 0; iterator->Valid() && rows < max_block_size; ++rows, iterator->Next()) { - ReadBufferFromString key_buffer(iterator->key()); - ReadBufferFromString value_buffer(iterator->value()); - fillColumns(key_buffer, value_buffer, columns); + fillColumns(iterator->key(), iterator->value(), primary_key_pos, getPort().getHeader(), columns); } if (!iterator->status().ok()) @@ -287,16 +323,6 @@ public: return Chunk(block.getColumns(), block.rows()); } - void fillColumns(ReadBufferFromString & key_buffer, ReadBufferFromString & value_buffer, MutableColumns & columns) - { - size_t idx = 0; - for (const auto & elem : getPort().getHeader()) - { - elem.type->getDefaultSerialization()->deserializeBinary(*columns[idx], idx == primary_key_pos ? key_buffer : value_buffer); - ++idx; - } - } - private: const StorageEmbeddedRocksDB & storage; @@ -321,7 +347,7 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, bool attach, ContextPtr context_, const String & primary_key_) - : IStorage(table_id_) + : IKeyValueStorage(table_id_) , WithContext(context_->getGlobalContext()) , primary_key{primary_key_} { @@ -427,6 +453,7 @@ void StorageEmbeddedRocksDB::initDB() throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to open rocksdb path at: {}: {}", rocksdb_dir, status.ToString()); } + /// It's ok just to wrap db with unique_ptr, from rdb documentation: "when you are done with a database, just delete the database object" rocksdb_ptr = std::unique_ptr(db); } @@ -527,6 +554,73 @@ std::vector StorageEmbeddedRocksDB::multiGet(const std::vector< return rocksdb_ptr->MultiGet(rocksdb::ReadOptions(), slices_keys, &values); } +Chunk StorageEmbeddedRocksDB::getByKeys( + const ColumnsWithTypeAndName & keys, + PaddedPODArray & null_map) const +{ + if (keys.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "StorageEmbeddedRocksDB supports only one key, got: {}", keys.size()); + + auto raw_keys = serializeKeysToRawString(keys[0]); + + if (raw_keys.size() != keys[0].column->size()) + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Assertion failed: {} != {}", raw_keys.size(), keys[0].column->size()); + + return getBySerializedKeys(raw_keys, &null_map); +} + +Chunk StorageEmbeddedRocksDB::getBySerializedKeys( + const std::vector & keys, + PaddedPODArray * null_map) const +{ + std::vector values; + Block sample_block = getInMemoryMetadataPtr()->getSampleBlock(); + + size_t primary_key_pos = getPrimaryKeyPos(sample_block, getPrimaryKey()); + + MutableColumns columns = sample_block.cloneEmptyColumns(); + + /// Convert from vector of string to vector of string refs (rocksdb::Slice), because multiGet api expects them. + std::vector slices_keys; + slices_keys.reserve(keys.size()); + for (const auto & key : keys) + slices_keys.emplace_back(key); + + + auto statuses = multiGet(slices_keys, values); + if (null_map) + { + null_map->clear(); + null_map->resize_fill(statuses.size(), 1); + } + + for (size_t i = 0; i < statuses.size(); ++i) + { + if (statuses[i].ok()) + { + fillColumns(slices_keys[i], values[i], primary_key_pos, sample_block, columns); + } + else if (statuses[i].IsNotFound()) + { + if (null_map) + { + (*null_map)[i] = 0; + for (size_t col_idx = 0; col_idx < sample_block.columns(); ++col_idx) + { + columns[col_idx]->insert(sample_block.getByPosition(col_idx).type->getDefault()); + } + } + } + else + { + throw DB::Exception(ErrorCodes::ROCKSDB_ERROR, "rocksdb error {}", statuses[i].ToString()); + } + } + + size_t num_rows = columns.at(0)->size(); + return Chunk(std::move(columns), num_rows); +} + void registerStorageEmbeddedRocksDB(StorageFactory & factory) { StorageFactory::StorageFeatures features{ diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h index ca6b4436c72..62c9a0eeae7 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -18,7 +19,11 @@ namespace DB class Context; -class StorageEmbeddedRocksDB final : public IStorage, WithContext +/// Wrapper for rocksdb storage. +/// Operates with rocksdb data structures via rocksdb API (holds pointer to rocksdb::DB inside for that). +/// Storage have one primary key. +/// Values are serialized into raw strings to store in rocksdb. +class StorageEmbeddedRocksDB final : public IKeyValueStorage, WithContext { friend class EmbeddedRocksDBSink; public: @@ -56,7 +61,16 @@ public: std::shared_ptr getRocksDBStatistics() const; std::vector multiGet(const std::vector & slices_keys, std::vector & values) const; - const String & getPrimaryKey() const { return primary_key; } + Names getPrimaryKey() const override { return {primary_key}; } + + Chunk getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray & null_map) const override; + + /// Return chunk with data for given serialized keys. + /// If out_null_map is passed, fill it with 1/0 depending on key was/wasn't found. Result chunk may contain default values. + /// If out_null_map is not passed. Not found rows excluded from result chunk. + Chunk getBySerializedKeys( + const std::vector & keys, + PaddedPODArray * out_null_map) const; private: const String primary_key; diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 4c962f36e4f..85fb20d6571 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -1079,8 +1080,8 @@ void registerStorageBuffer(StorageFactory & factory) size_t i = 0; - String destination_database = engine_args[i++]->as().value.safeGet(); - String destination_table = engine_args[i++]->as().value.safeGet(); + String destination_database = checkAndGetLiteralArgument(engine_args[i++], "destination_database"); + String destination_table = checkAndGetLiteralArgument(engine_args[i++], "destination_table"); UInt64 num_buckets = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index fda6da6c1ff..2839ac03a5b 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB @@ -339,7 +340,7 @@ void registerStorageDictionary(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); args.engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args.engine_args[0], local_context); - String dictionary_name = args.engine_args[0]->as().value.safeGet(); + String dictionary_name = checkAndGetLiteralArgument(args.engine_args[0], "dictionary_name"); if (!args.attach) { diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 1c785df9be4..03eb400a8ad 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -34,10 +35,6 @@ #include #include #include -#include -#include -#include -#include #include #include @@ -1437,15 +1434,15 @@ void registerStorageDistributed(StorageFactory & factory) engine_args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[1], local_context); engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], local_context); - String remote_database = engine_args[1]->as().value.safeGet(); - String remote_table = engine_args[2]->as().value.safeGet(); + String remote_database = checkAndGetLiteralArgument(engine_args[1], "remote_database"); + String remote_table = checkAndGetLiteralArgument(engine_args[2], "remote_table"); const auto & sharding_key = engine_args.size() >= 4 ? engine_args[3] : nullptr; String storage_policy = "default"; if (engine_args.size() >= 5) { engine_args[4] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[4], local_context); - storage_policy = engine_args[4]->as().value.safeGet(); + storage_policy = checkAndGetLiteralArgument(engine_args[4], "storage_policy"); } /// Check that sharding_key exists in the table and has numeric type. diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index e0cbdbe98af..2931e62b7ef 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace DB @@ -179,14 +180,14 @@ void registerStorageExecutable(StorageFactory & factory) for (size_t i = 0; i < 2; ++i) args.engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args.engine_args[i], local_context); - auto scipt_name_with_arguments_value = args.engine_args[0]->as().value.safeGet(); + auto script_name_with_arguments_value = checkAndGetLiteralArgument(args.engine_args[0], "script_name_with_arguments_value"); std::vector script_name_with_arguments; - boost::split(script_name_with_arguments, scipt_name_with_arguments_value, [](char c) { return c == ' '; }); + boost::split(script_name_with_arguments, script_name_with_arguments_value, [](char c) { return c == ' '; }); auto script_name = script_name_with_arguments[0]; script_name_with_arguments.erase(script_name_with_arguments.begin()); - auto format = args.engine_args[1]->as().value.safeGet(); + auto format = checkAndGetLiteralArgument(args.engine_args[1], "format"); std::vector input_queries; for (size_t i = 2; i < args.engine_args.size(); ++i) diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 181cf0ca183..dcb7a90b2f6 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -3,13 +3,9 @@ #include #include -#include -#include #include -#include #include #include -#include #include #include #include @@ -17,6 +13,7 @@ #include #include #include +#include #include #include @@ -95,10 +92,13 @@ StorageExternalDistributed::StorageExternalDistributed( postgres_conf.set(configuration); postgres_conf.addresses = addresses; + const auto & settings = context->getSettingsRef(); auto pool = std::make_shared( postgres_conf, - context->getSettingsRef().postgresql_connection_pool_size, - context->getSettingsRef().postgresql_connection_pool_wait_timeout); + settings.postgresql_connection_pool_size, + settings.postgresql_connection_pool_wait_timeout, + POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + settings.postgresql_connection_pool_auto_close_connection); shard = std::make_shared(table_id_, std::move(pool), configuration.table, columns_, constraints_, String{}); break; @@ -229,7 +229,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) if (engine_args.size() < 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine ExternalDistributed must have at least 2 arguments: engine_name, named_collection and/or description"); - auto engine_name = engine_args[0]->as().value.safeGet(); + auto engine_name = checkAndGetLiteralArgument(engine_args[0], "engine_name"); StorageExternalDistributed::ExternalStorageEngine table_engine; if (engine_name == "URL") table_engine = StorageExternalDistributed::ExternalStorageEngine::URL; @@ -256,7 +256,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) for (const auto & [name, value] : storage_specific_args) { if (name == "description") - cluster_description = value->as()->value.safeGet(); + cluster_description = checkAndGetLiteralArgument(value, "cluster_description"); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown key-value argument {} for table engine URL", name); @@ -271,11 +271,11 @@ void registerStorageExternalDistributed(StorageFactory & factory) for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext()); - cluster_description = engine_args[1]->as().value.safeGet(); - configuration.format = engine_args[2]->as().value.safeGet(); + cluster_description = checkAndGetLiteralArgument(engine_args[1], "cluster_description"); + configuration.format = checkAndGetLiteralArgument(engine_args[2], "format"); configuration.compression_method = "auto"; if (engine_args.size() == 4) - configuration.compression_method = engine_args[3]->as().value.safeGet(); + configuration.compression_method = checkAndGetLiteralArgument(engine_args[3], "compression_method"); } @@ -302,7 +302,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) for (const auto & [name, value] : storage_specific_args) { if (name == "description") - cluster_description = value->as()->value.safeGet(); + cluster_description = checkAndGetLiteralArgument(value, "cluster_description"); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown key-value argument {} for table function URL", name); @@ -320,11 +320,11 @@ void registerStorageExternalDistributed(StorageFactory & factory) "ExternalDistributed('engine_name', 'cluster_description', 'database', 'table', 'user', 'password').", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - cluster_description = engine_args[1]->as().value.safeGet(); - configuration.database = engine_args[2]->as().value.safeGet(); - configuration.table = engine_args[3]->as().value.safeGet(); - configuration.username = engine_args[4]->as().value.safeGet(); - configuration.password = engine_args[5]->as().value.safeGet(); + cluster_description = checkAndGetLiteralArgument(engine_args[1], "cluster_description"); + configuration.database = checkAndGetLiteralArgument(engine_args[2], "database"); + configuration.table = checkAndGetLiteralArgument(engine_args[3], "table"); + configuration.username = checkAndGetLiteralArgument(engine_args[4], "username"); + configuration.password = checkAndGetLiteralArgument(engine_args[5], "password"); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 2fa6003c0eb..d138104018a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -1,5 +1,10 @@ #include #include +#include +#include +#include +#include +#include #include #include @@ -20,30 +25,26 @@ #include #include #include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include #include #include #include #include #include +#include + +#include +#include +#include +#include + #include #include -#include + +#include +#include +#include +#include +#include namespace fs = std::filesystem; @@ -1103,7 +1104,7 @@ void registerStorageFile(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); engine_args_ast[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[0], factory_args.getLocalContext()); - storage_args.format_name = engine_args_ast[0]->as().value.safeGet(); + storage_args.format_name = checkAndGetLiteralArgument(engine_args_ast[0], "format_name"); // Use format settings from global server context + settings from // the SETTINGS clause of the create query. Settings from current @@ -1171,7 +1172,7 @@ void registerStorageFile(StorageFactory & factory) if (engine_args_ast.size() == 3) { engine_args_ast[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args_ast[2], factory_args.getLocalContext()); - storage_args.compression_method = engine_args_ast[2]->as().value.safeGet(); + storage_args.compression_method = checkAndGetLiteralArgument(engine_args_ast[2], "compression_method"); } else storage_args.compression_method = "auto"; diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index fa0baea40cd..d875b4ee80c 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -469,16 +469,16 @@ void registerStorageGenerateRandom(StorageFactory & factory) if (!engine_args.empty()) { - const Field & value = engine_args[0]->as().value; - if (!value.isNull()) - random_seed = value.safeGet(); + const auto & ast_literal = engine_args[0]->as(); + if (!ast_literal.value.isNull()) + random_seed = checkAndGetLiteralArgument(ast_literal, "random_seed"); } if (engine_args.size() >= 2) - max_string_length = engine_args[1]->as().value.safeGet(); + max_string_length = checkAndGetLiteralArgument(engine_args[1], "max_string_length"); if (engine_args.size() == 3) - max_array_length = engine_args[2]->as().value.safeGet(); + max_array_length = checkAndGetLiteralArgument(engine_args[2], "max_array_length"); return std::make_shared(args.table_id, args.columns, args.comment, max_array_length, max_string_length, random_seed); }); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index f6d7e8e7afd..0afc7a0df7e 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1,9 +1,10 @@ #include -#include +#include #include #include #include #include +#include #include #include #include @@ -22,17 +23,16 @@ #include #include #include -#include "Processors/QueryPlan/BuildQueryPipelineSettings.h" -#include "Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h" -#include -#include -#include -#include +#include +#include #include #include #include #include -#include +#include +#include +#include +#include namespace DB @@ -848,7 +848,7 @@ std::tuple StorageMerge::evaluateDatabaseName(cons throw Exception("REGEXP in Merge ENGINE takes only one argument", ErrorCodes::BAD_ARGUMENTS); auto * literal = func->arguments->children[0]->as(); - if (!literal || literal->value.safeGet().empty()) + if (!literal || literal->value.getType() != Field::Types::Which::String || literal->value.safeGet().empty()) throw Exception("Argument for REGEXP in Merge ENGINE should be a non empty String Literal", ErrorCodes::BAD_ARGUMENTS); return {true, func->arguments->children[0]}; @@ -879,10 +879,10 @@ void registerStorageMerge(StorageFactory & factory) if (!is_regexp) engine_args[0] = database_ast; - String source_database_name_or_regexp = database_ast->as().value.safeGet(); + String source_database_name_or_regexp = checkAndGetLiteralArgument(database_ast, "database_name"); engine_args[1] = evaluateConstantExpressionAsLiteral(engine_args[1], args.getLocalContext()); - String table_name_regexp = engine_args[1]->as().value.safeGet(); + String table_name_regexp = checkAndGetLiteralArgument(engine_args[1], "table_name_regexp"); return std::make_shared( args.table_id, args.columns, args.comment, source_database_name_or_regexp, is_regexp, table_name_regexp, args.getContext()); diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 11a1f8ba4d6..1f2523c8645 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -1,11 +1,12 @@ -#include "StorageMongoDB.h" -#include "StorageMongoDBSocketFactory.h" +#include +#include +#include +#include #include #include #include #include -#include #include #include #include @@ -120,7 +121,7 @@ StorageMongoDBConfiguration StorageMongoDB::getConfiguration(ASTs engine_args, C for (const auto & [arg_name, arg_value] : storage_specific_args) { if (arg_name == "options") - configuration.options = arg_value->as()->value.safeGet(); + configuration.options = checkAndGetLiteralArgument(arg_value, "options"); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected key-value argument." @@ -139,17 +140,17 @@ StorageMongoDBConfiguration StorageMongoDB::getConfiguration(ASTs engine_args, C engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); /// 27017 is the default MongoDB port. - auto parsed_host_port = parseAddress(engine_args[0]->as().value.safeGet(), 27017); + auto parsed_host_port = parseAddress(checkAndGetLiteralArgument(engine_args[0], "host:port"), 27017); configuration.host = parsed_host_port.first; configuration.port = parsed_host_port.second; - configuration.database = engine_args[1]->as().value.safeGet(); - configuration.table = engine_args[2]->as().value.safeGet(); - configuration.username = engine_args[3]->as().value.safeGet(); - configuration.password = engine_args[4]->as().value.safeGet(); + configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); + configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); + configuration.username = checkAndGetLiteralArgument(engine_args[3], "username"); + configuration.password = checkAndGetLiteralArgument(engine_args[4], "password"); if (engine_args.size() >= 6) - configuration.options = engine_args[5]->as().value.safeGet(); + configuration.options = checkAndGetLiteralArgument(engine_args[5], "database"); } diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 3ed97712292..7fe008eead4 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -5,14 +5,12 @@ #include #include #include +#include #include #include -#include -#include #include #include #include -#include #include #include #include @@ -253,9 +251,9 @@ StorageMySQLConfiguration StorageMySQL::getConfiguration(ASTs engine_args, Conte for (const auto & [arg_name, arg_value] : storage_specific_args) { if (arg_name == "replace_query") - configuration.replace_query = arg_value->as()->value.safeGet(); + configuration.replace_query = checkAndGetLiteralArgument(arg_value, "replace_query"); else if (arg_name == "on_duplicate_clause") - configuration.on_duplicate_clause = arg_value->as()->value.safeGet(); + configuration.on_duplicate_clause = checkAndGetLiteralArgument(arg_value, "on_duplicate_clause"); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected key-value argument." @@ -273,18 +271,18 @@ StorageMySQLConfiguration StorageMySQL::getConfiguration(ASTs engine_args, Conte for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context_); - const auto & host_port = engine_args[0]->as().value.safeGet(); + const auto & host_port = checkAndGetLiteralArgument(engine_args[0], "host:port"); size_t max_addresses = context_->getSettingsRef().glob_expansion_max_elements; configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); - configuration.database = engine_args[1]->as().value.safeGet(); - configuration.table = engine_args[2]->as().value.safeGet(); - configuration.username = engine_args[3]->as().value.safeGet(); - configuration.password = engine_args[4]->as().value.safeGet(); + configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); + configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); + configuration.username = checkAndGetLiteralArgument(engine_args[3], "username"); + configuration.password = checkAndGetLiteralArgument(engine_args[4], "password"); if (engine_args.size() >= 6) - configuration.replace_query = engine_args[5]->as().value.safeGet(); + configuration.replace_query = checkAndGetLiteralArgument(engine_args[5], "replace_query"); if (engine_args.size() == 7) - configuration.on_duplicate_clause = engine_args[6]->as().value.safeGet(); + configuration.on_duplicate_clause = checkAndGetLiteralArgument(engine_args[6], "on_duplicate_clause"); } for (const auto & address : configuration.addresses) context_->getRemoteHostFilter().checkHostAndPort(address.first, toString(address.second)); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 5b57384c1dd..e0c6dbf5463 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -11,8 +11,6 @@ #include #include -#include -#include #include #include @@ -31,7 +29,6 @@ #include #include -#include #include #include @@ -40,6 +37,7 @@ #include #include +#include namespace DB @@ -400,7 +398,7 @@ StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_a for (const auto & [arg_name, arg_value] : storage_specific_args) { if (arg_name == "on_conflict") - configuration.on_conflict = arg_value->as()->value.safeGet(); + configuration.on_conflict = checkAndGetLiteralArgument(arg_value, "on_conflict"); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected key-value argument." @@ -418,7 +416,7 @@ StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_a for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); - const auto & host_port = engine_args[0]->as().value.safeGet(); + const auto & host_port = checkAndGetLiteralArgument(engine_args[0], "host:port"); size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 5432); @@ -427,15 +425,15 @@ StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_a configuration.host = configuration.addresses[0].first; configuration.port = configuration.addresses[0].second; } - configuration.database = engine_args[1]->as().value.safeGet(); - configuration.table = engine_args[2]->as().value.safeGet(); - configuration.username = engine_args[3]->as().value.safeGet(); - configuration.password = engine_args[4]->as().value.safeGet(); + configuration.database = checkAndGetLiteralArgument(engine_args[1], "host:port"); + configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); + configuration.username = checkAndGetLiteralArgument(engine_args[3], "username"); + configuration.password = checkAndGetLiteralArgument(engine_args[4], "password"); if (engine_args.size() >= 6) - configuration.schema = engine_args[5]->as().value.safeGet(); + configuration.schema = checkAndGetLiteralArgument(engine_args[5], "schema"); if (engine_args.size() >= 7) - configuration.on_conflict = engine_args[6]->as().value.safeGet(); + configuration.on_conflict = checkAndGetLiteralArgument(engine_args[6], "on_conflict"); } for (const auto & address : configuration.addresses) context->getRemoteHostFilter().checkHostAndPort(address.first, toString(address.second)); @@ -449,9 +447,12 @@ void registerStoragePostgreSQL(StorageFactory & factory) factory.registerStorage("PostgreSQL", [](const StorageFactory::Arguments & args) { auto configuration = StoragePostgreSQL::getConfiguration(args.engine_args, args.getLocalContext()); + const auto & settings = args.getContext()->getSettingsRef(); auto pool = std::make_shared(configuration, - args.getContext()->getSettingsRef().postgresql_connection_pool_size, - args.getContext()->getSettingsRef().postgresql_connection_pool_wait_timeout); + settings.postgresql_connection_pool_size, + settings.postgresql_connection_pool_wait_timeout, + POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + settings.postgresql_connection_pool_auto_close_connection); return std::make_shared( args.table_id, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a7d9641d5c4..bed21a9affc 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -11,14 +11,12 @@ #include -#include #include #include #include #include #include -#include #include #include @@ -27,6 +25,7 @@ #include #include #include +#include #include #include @@ -1051,25 +1050,25 @@ void StorageS3::processNamedCollectionResult(StorageS3Configuration & configurat for (const auto & [arg_name, arg_value] : key_value_args) { if (arg_name == "access_key_id") - configuration.auth_settings.access_key_id = arg_value->as()->value.safeGet(); + configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(arg_value, "access_key_id"); else if (arg_name == "secret_access_key") - configuration.auth_settings.secret_access_key = arg_value->as()->value.safeGet(); + configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(arg_value, "secret_access_key"); else if (arg_name == "filename") - configuration.url = std::filesystem::path(configuration.url) / arg_value->as()->value.safeGet(); + configuration.url = std::filesystem::path(configuration.url) / checkAndGetLiteralArgument(arg_value, "filename"); else if (arg_name == "use_environment_credentials") - configuration.auth_settings.use_environment_credentials = arg_value->as()->value.safeGet(); + configuration.auth_settings.use_environment_credentials = checkAndGetLiteralArgument(arg_value, "use_environment_credentials"); else if (arg_name == "max_single_read_retries") - configuration.rw_settings.max_single_read_retries = arg_value->as()->value.safeGet(); + configuration.rw_settings.max_single_read_retries = checkAndGetLiteralArgument(arg_value, "max_single_read_retries"); else if (arg_name == "min_upload_part_size") - configuration.rw_settings.max_single_read_retries = arg_value->as()->value.safeGet(); + configuration.rw_settings.max_single_read_retries = checkAndGetLiteralArgument(arg_value, "min_upload_part_size"); else if (arg_name == "upload_part_size_multiply_factor") - configuration.rw_settings.max_single_read_retries = arg_value->as()->value.safeGet(); + configuration.rw_settings.max_single_read_retries = checkAndGetLiteralArgument(arg_value, "upload_part_size_multiply_factor"); else if (arg_name == "upload_part_size_multiply_parts_count_threshold") - configuration.rw_settings.max_single_read_retries = arg_value->as()->value.safeGet(); + configuration.rw_settings.max_single_read_retries = checkAndGetLiteralArgument(arg_value, "upload_part_size_multiply_parts_count_threshold"); else if (arg_name == "max_single_part_upload_size") - configuration.rw_settings.max_single_read_retries = arg_value->as()->value.safeGet(); + configuration.rw_settings.max_single_read_retries = checkAndGetLiteralArgument(arg_value, "max_single_part_upload_size"); else if (arg_name == "max_connections") - configuration.rw_settings.max_single_read_retries = arg_value->as()->value.safeGet(); + configuration.rw_settings.max_single_read_retries = checkAndGetLiteralArgument(arg_value, "max_connections"); else throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Unknown key-value argument `{}` for StorageS3, expected: url, [access_key_id, secret_access_key], name of used format and [compression_method].", @@ -1098,22 +1097,22 @@ StorageS3Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPt for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context); - configuration.url = engine_args[0]->as().value.safeGet(); + configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); if (engine_args.size() >= 4) { - configuration.auth_settings.access_key_id = engine_args[1]->as().value.safeGet(); - configuration.auth_settings.secret_access_key = engine_args[2]->as().value.safeGet(); + configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); + configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); } if (engine_args.size() == 3 || engine_args.size() == 5) { - configuration.compression_method = engine_args.back()->as().value.safeGet(); - configuration.format = engine_args[engine_args.size() - 2]->as().value.safeGet(); + configuration.compression_method = checkAndGetLiteralArgument(engine_args.back(), "compression_method"); + configuration.format = checkAndGetLiteralArgument(engine_args[engine_args.size() - 2], "format"); } else if (engine_args.size() != 1) { configuration.compression_method = "auto"; - configuration.format = engine_args.back()->as().value.safeGet(); + configuration.format = checkAndGetLiteralArgument(engine_args.back(), "format"); } } diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index 1eb473af80d..a86ed7646b3 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -1,12 +1,10 @@ #include "StorageSQLite.h" #if USE_SQLITE -#include #include #include #include #include -#include #include #include #include @@ -16,6 +14,7 @@ #include #include #include +#include #include #include @@ -168,8 +167,8 @@ void registerStorageSQLite(StorageFactory & factory) for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext()); - const auto database_path = engine_args[0]->as().value.safeGet(); - const auto table_name = engine_args[1]->as().value.safeGet(); + const auto database_path = checkAndGetLiteralArgument(engine_args[0], "database_path"); + const auto table_name = checkAndGetLiteralArgument(engine_args[1], "table_name"); auto sqlite_db = openSQLiteDB(database_path, args.getContext(), /* throw_on_error */!args.attach); diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index ad63499acfa..2f586a3c26c 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -11,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp index d935d73d03d..b47623db50b 100644 --- a/src/Storages/StorageSnapshot.cpp +++ b/src/Storages/StorageSnapshot.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include namespace DB diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 2033d33a33d..e25db92be64 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -1,8 +1,6 @@ #include #include -#include -#include #include #include @@ -12,7 +10,6 @@ #include #include #include -#include #include #include @@ -21,11 +18,8 @@ #include -#include - #include -#include #include #include #include diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index cbec49865a1..15ae23305f3 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1,6 +1,8 @@ #include +#include +#include +#include -#include #include #include #include @@ -21,17 +23,15 @@ #include #include #include - -#include -#include -#include "Common/ThreadStatus.h" -#include -#include "IO/HTTPCommon.h" -#include "IO/ReadWriteBufferFromHTTP.h" - -#include #include #include + +#include +#include +#include +#include + +#include #include #include #include @@ -960,11 +960,11 @@ URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, Contex if (header_it != args.end()) args.erase(header_it); - configuration.url = args[0]->as().value.safeGet(); + configuration.url = checkAndGetLiteralArgument(args[0], "url"); if (args.size() > 1) - configuration.format = args[1]->as().value.safeGet(); + configuration.format = checkAndGetLiteralArgument(args[1], "format"); if (args.size() == 3) - configuration.compression_method = args[2]->as().value.safeGet(); + configuration.compression_method = checkAndGetLiteralArgument(args[2], "compression_method"); } if (configuration.format == "auto") diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index f44daf2557e..0b7a1ae75d4 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -1,16 +1,16 @@ -#include "StorageXDBC.h" +#include +#include +#include +#include +#include #include -#include #include #include #include #include #include #include -#include -#include -#include #include #include @@ -173,11 +173,11 @@ namespace BridgeHelperPtr bridge_helper = std::make_shared>(args.getContext(), args.getContext()->getSettingsRef().http_receive_timeout.value, - engine_args[0]->as().value.safeGet()); + checkAndGetLiteralArgument(engine_args[0], "connection_string")); return std::make_shared( args.table_id, - engine_args[1]->as().value.safeGet(), - engine_args[2]->as().value.safeGet(), + checkAndGetLiteralArgument(engine_args[1], "database_name"), + checkAndGetLiteralArgument(engine_args[2], "table_name"), args.columns, args.comment, args.getContext(), diff --git a/src/Storages/RabbitMQ/UVLoop.h b/src/Storages/UVLoop.h similarity index 94% rename from src/Storages/RabbitMQ/UVLoop.h rename to src/Storages/UVLoop.h index 4de67cbc206..66668739dd7 100644 --- a/src/Storages/RabbitMQ/UVLoop.h +++ b/src/Storages/UVLoop.h @@ -2,8 +2,8 @@ #include -#include #include +#include #include @@ -19,7 +19,7 @@ namespace ErrorCodes class UVLoop : public boost::noncopyable { public: - UVLoop(): loop_ptr(new uv_loop_t()) + UVLoop() : loop_ptr(new uv_loop_t()) { int res = uv_loop_init(loop_ptr.get()); diff --git a/src/Storages/checkAndGetLiteralArgument.cpp b/src/Storages/checkAndGetLiteralArgument.cpp new file mode 100644 index 00000000000..3c43ce98920 --- /dev/null +++ b/src/Storages/checkAndGetLiteralArgument.cpp @@ -0,0 +1,40 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +template +T checkAndGetLiteralArgument(const ASTPtr & arg, const String & arg_name) +{ + return checkAndGetLiteralArgument(*arg->as(), arg_name); +} + +template +T checkAndGetLiteralArgument(const ASTLiteral & arg, const String & arg_name) +{ + auto requested_type = Field::TypeToEnum>>::value; + auto provided_type = arg.value.getType(); + if (requested_type != provided_type) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Argument '{}' must be a literal with type {}, got {}", + arg_name, + fieldTypeToString(requested_type), + fieldTypeToString(provided_type)); + + return arg.value.safeGet(); +} + +template String checkAndGetLiteralArgument(const ASTPtr &, const String &); +template UInt64 checkAndGetLiteralArgument(const ASTPtr &, const String &); +template UInt8 checkAndGetLiteralArgument(const ASTPtr &, const String &); +template bool checkAndGetLiteralArgument(const ASTPtr &, const String &); +template String checkAndGetLiteralArgument(const ASTLiteral &, const String &); + +} diff --git a/src/Storages/checkAndGetLiteralArgument.h b/src/Storages/checkAndGetLiteralArgument.h new file mode 100644 index 00000000000..086deca5121 --- /dev/null +++ b/src/Storages/checkAndGetLiteralArgument.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace DB +{ + +template +T checkAndGetLiteralArgument(const ASTPtr & arg, const String & arg_name); + +template +T checkAndGetLiteralArgument(const ASTLiteral & arg, const String & arg_name); + +} diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 585e85688fc..575b3de7ae2 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -54,6 +54,7 @@ void registerStorageMySQL(StorageFactory & factory); void registerStorageMongoDB(StorageFactory & factory); + #if USE_RDKAFKA void registerStorageKafka(StorageFactory & factory); #endif @@ -62,6 +63,10 @@ void registerStorageKafka(StorageFactory & factory); void registerStorageRabbitMQ(StorageFactory & factory); #endif +#if USE_NATSIO +void registerStorageNATS(StorageFactory & factory); +#endif + #if USE_ROCKSDB void registerStorageEmbeddedRocksDB(StorageFactory & factory); #endif @@ -146,6 +151,10 @@ void registerStorages() registerStorageRabbitMQ(factory); #endif + #if USE_NATSIO + registerStorageNATS(factory); + #endif + #if USE_ROCKSDB registerStorageEmbeddedRocksDB(factory); #endif diff --git a/src/TableFunctions/Hive/TableFunctionHive.cpp b/src/TableFunctions/Hive/TableFunctionHive.cpp index 99dded030e5..12371df4e3c 100644 --- a/src/TableFunctions/Hive/TableFunctionHive.cpp +++ b/src/TableFunctions/Hive/TableFunctionHive.cpp @@ -2,11 +2,9 @@ #if USE_HIVE #include -#include #include #include #include -#include #include #include #include @@ -14,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -44,11 +43,11 @@ namespace DB for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context_); - hive_metastore_url = args[0]->as().value.safeGet(); - hive_database = args[1]->as().value.safeGet(); - hive_table = args[2]->as().value.safeGet(); - table_structure = args[3]->as().value.safeGet(); - partition_by_def = args[4]->as().value.safeGet(); + hive_metastore_url = checkAndGetLiteralArgument(args[0], "hive_url"); + hive_database = checkAndGetLiteralArgument(args[1], "hive_database"); + hive_table = checkAndGetLiteralArgument(args[2], "hive_table"); + table_structure = checkAndGetLiteralArgument(args[3], "structure"); + partition_by_def = checkAndGetLiteralArgument(args[4], "partition_by_keys"); actual_columns = parseColumnsListFromString(table_structure, context_); } diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 7fa3ccda195..e2391787726 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -8,11 +8,10 @@ #include #include +#include #include -#include - #include namespace DB @@ -25,10 +24,9 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -void ITableFunctionFileLike::parseFirstArguments(const ASTPtr & arg, ContextPtr context) +void ITableFunctionFileLike::parseFirstArguments(const ASTPtr & arg, const ContextPtr &) { - auto ast = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - filename = ast->as().value.safeGet(); + filename = checkAndGetLiteralArgument(arg, "source"); } String ITableFunctionFileLike::getFormatFromFirstArgument() @@ -49,13 +47,13 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context if (args.empty()) throw Exception("Table function '" + getName() + "' requires at least 1 argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + parseFirstArguments(args[0], context); - for (size_t i = 1; i < args.size(); ++i) - args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); - if (args.size() > 1) - format = args[1]->as().value.safeGet(); + format = checkAndGetLiteralArgument(args[1], "format"); if (format == "auto") format = getFormatFromFirstArgument(); @@ -67,7 +65,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context throw Exception("Table function '" + getName() + "' requires 1, 2, 3 or 4 arguments: filename, format (default auto), structure (default auto) and compression method (default auto)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - structure = args[2]->as().value.safeGet(); + structure = checkAndGetLiteralArgument(args[2], "structure"); if (structure.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -75,7 +73,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context ast_function->formatForErrorMessage()); if (args.size() == 4) - compression_method = args[3]->as().value.safeGet(); + compression_method = checkAndGetLiteralArgument(args[3], "compression_method"); } StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 88ad75b1018..c2f32eb0aa3 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -20,7 +20,7 @@ public: protected: void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - virtual void parseFirstArguments(const ASTPtr & arg, ContextPtr context); + virtual void parseFirstArguments(const ASTPtr & arg, const ContextPtr & context); virtual String getFormatFromFirstArgument(); String filename; diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp index c251b2703e1..54c23cfb64b 100644 --- a/src/TableFunctions/TableFunctionDictionary.cpp +++ b/src/TableFunctions/TableFunctionDictionary.cpp @@ -7,6 +7,7 @@ #include #include +#include #include @@ -35,7 +36,7 @@ void TableFunctionDictionary::parseArguments(const ASTPtr & ast_function, Contex for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - dictionary_name = args[0]->as().value.safeGet(); + dictionary_name = checkAndGetLiteralArgument(args[0], "dictionary_name"); } ColumnsDescription TableFunctionDictionary::getActualTableStructure(ContextPtr context) const diff --git a/src/TableFunctions/TableFunctionExecutable.cpp b/src/TableFunctions/TableFunctionExecutable.cpp index dc88cca51e6..b84008f5ac8 100644 --- a/src/TableFunctions/TableFunctionExecutable.cpp +++ b/src/TableFunctions/TableFunctionExecutable.cpp @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -43,16 +43,16 @@ void TableFunctionExecutable::parseArguments(const ASTPtr & ast_function, Contex for (size_t i = 0; i <= 2; ++i) args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); - auto scipt_name_with_arguments_value = args[0]->as().value.safeGet(); + auto script_name_with_arguments_value = checkAndGetLiteralArgument(args[0], "script_name_with_arguments_value"); std::vector script_name_with_arguments; - boost::split(script_name_with_arguments, scipt_name_with_arguments_value, [](char c){ return c == ' '; }); + boost::split(script_name_with_arguments, script_name_with_arguments_value, [](char c){ return c == ' '; }); script_name = script_name_with_arguments[0]; script_name_with_arguments.erase(script_name_with_arguments.begin()); arguments = std::move(script_name_with_arguments); - format = args[1]->as().value.safeGet(); - structure = args[2]->as().value.safeGet(); + format = checkAndGetLiteralArgument(args[1], "format"); + structure = checkAndGetLiteralArgument(args[2], "structure"); for (size_t i = 3; i < args.size(); ++i) { diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 507b3406cb8..6f8f0db46a0 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, ContextPtr context) +void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, const ContextPtr & context) { if (context->getApplicationType() != Context::ApplicationType::LOCAL) { @@ -29,36 +29,27 @@ void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, ContextPtr conte return; } - if (auto opt_name = tryGetIdentifierName(arg)) + const auto * literal = arg->as(); + auto type = literal->value.getType(); + if (type == Field::Types::String) { - if (*opt_name == "stdin") + filename = literal->value.safeGet(); + if (filename == "stdin" || filename == "-") fd = STDIN_FILENO; - else if (*opt_name == "stdout") + else if (filename == "stdout") fd = STDOUT_FILENO; - else if (*opt_name == "stderr") + else if (filename == "stderr") fd = STDERR_FILENO; - else - filename = *opt_name; } - else if (const auto * literal = arg->as()) + else if (type == Field::Types::Int64 || type == Field::Types::UInt64) { - auto type = literal->value.getType(); - if (type == Field::Types::Int64 || type == Field::Types::UInt64) - { - fd = (type == Field::Types::Int64) ? static_cast(literal->value.get()) : static_cast(literal->value.get()); - if (fd < 0) - throw Exception("File descriptor must be non-negative", ErrorCodes::BAD_ARGUMENTS); - } - else if (type == Field::Types::String) - { - filename = literal->value.get(); - if (filename == "-") - fd = STDIN_FILENO; - } - else - throw Exception( - "The first argument of table function '" + getName() + "' mush be path or file descriptor", ErrorCodes::BAD_ARGUMENTS); + fd = (type == Field::Types::Int64) ? literal->value.get() : literal->value.get(); + if (fd < 0) + throw Exception("File descriptor must be non-negative", ErrorCodes::BAD_ARGUMENTS); } + else + throw Exception( + "The first argument of table function '" + getName() + "' mush be path or file descriptor", ErrorCodes::BAD_ARGUMENTS); } String TableFunctionFile::getFormatFromFirstArgument() diff --git a/src/TableFunctions/TableFunctionFile.h b/src/TableFunctions/TableFunctionFile.h index f956043e69a..20ecdb6222c 100644 --- a/src/TableFunctions/TableFunctionFile.h +++ b/src/TableFunctions/TableFunctionFile.h @@ -24,7 +24,7 @@ public: protected: int fd = -1; - void parseFirstArguments(const ASTPtr & arg, ContextPtr context) override; + void parseFirstArguments(const ASTPtr & arg, const ContextPtr & context) override; String getFormatFromFirstArgument() override; private: diff --git a/src/TableFunctions/TableFunctionFormat.cpp b/src/TableFunctions/TableFunctionFormat.cpp index d3ce9627598..d47f8353e18 100644 --- a/src/TableFunctions/TableFunctionFormat.cpp +++ b/src/TableFunctions/TableFunctionFormat.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -43,8 +44,8 @@ void TableFunctionFormat::parseArguments(const ASTPtr & ast_function, ContextPtr for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - format = args[0]->as().value.safeGet(); - data = args[1]->as().value.safeGet(); + format = checkAndGetLiteralArgument(args[0], "format"); + data = checkAndGetLiteralArgument(args[1], "data"); } ColumnsDescription TableFunctionFormat::getActualTableStructure(ContextPtr context) const diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index ad766c6c66e..083e4a54190 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -58,20 +59,20 @@ void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, Co } /// Parsing first argument as table structure and creating a sample block - structure = args[0]->as().value.safeGet(); + structure = checkAndGetLiteralArgument(args[0], "structure"); if (args.size() >= 2) { - const Field & value = args[1]->as().value; - if (!value.isNull()) - random_seed = value.safeGet(); + const auto & literal = args[1]->as(); + if (!literal.value.isNull()) + random_seed = checkAndGetLiteralArgument(literal, "random_seed"); } if (args.size() >= 3) - max_string_length = args[2]->as().value.safeGet(); + max_string_length = checkAndGetLiteralArgument(args[2], "max_string_length"); if (args.size() == 4) - max_array_length = args[3]->as().value.safeGet(); + max_array_length = checkAndGetLiteralArgument(args[3], "max_string_length"); } ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(ContextPtr context) const diff --git a/src/TableFunctions/TableFunctionHDFSCluster.cpp b/src/TableFunctions/TableFunctionHDFSCluster.cpp index 80f19cd015a..b5e14a91b91 100644 --- a/src/TableFunctions/TableFunctionHDFSCluster.cpp +++ b/src/TableFunctions/TableFunctionHDFSCluster.cpp @@ -5,8 +5,8 @@ #include #include -#include #include +#include #include #include #include @@ -15,8 +15,6 @@ #include #include #include -#include -#include #include #include "registerTableFunctions.h" @@ -61,7 +59,7 @@ void TableFunctionHDFSCluster::parseArguments(const ASTPtr & ast_function, Conte arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); /// This argument is always the first - cluster_name = args[0]->as().value.safeGet(); + cluster_name = checkAndGetLiteralArgument(args[0], "cluster_name"); if (!context->tryGetCluster(cluster_name)) throw Exception(ErrorCodes::BAD_GET, "Requested cluster '{}' not found", cluster_name); diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index 0ff56fefb68..0f26cab3683 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,6 +5,7 @@ #include #include #include +#include #include #include #include "registerTableFunctions.h" @@ -40,7 +40,7 @@ void TableFunctionInput::parseArguments(const ASTPtr & ast_function, ContextPtr throw Exception("Table function '" + getName() + "' requires exactly 1 argument: structure", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - structure = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context)->as().value.safeGet(); + structure = checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context), "structure"); } ColumnsDescription TableFunctionInput::getActualTableStructure(ContextPtr context) const diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 28aed2f03ed..b055e241459 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -58,10 +59,10 @@ void TableFunctionMerge::parseArguments(const ASTPtr & ast_function, ContextPtr if (!is_regexp) args[0] = database_ast; - source_database_name_or_regexp = database_ast->as().value.safeGet(); + source_database_name_or_regexp = checkAndGetLiteralArgument(database_ast, "database_name"); args[1] = evaluateConstantExpressionAsLiteral(args[1], context); - source_table_regexp = args[1]->as().value.safeGet(); + source_table_regexp = checkAndGetLiteralArgument(args[1], "table_name_regexp"); } diff --git a/src/TableFunctions/TableFunctionNull.cpp b/src/TableFunctions/TableFunctionNull.cpp index dea95b86ffd..f5d5a92ec1a 100644 --- a/src/TableFunctions/TableFunctionNull.cpp +++ b/src/TableFunctions/TableFunctionNull.cpp @@ -1,9 +1,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -30,7 +30,7 @@ void TableFunctionNull::parseArguments(const ASTPtr & ast_function, ContextPtr c ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!arguments.empty()) - structure = evaluateConstantExpressionOrIdentifierAsLiteral(arguments[0], context)->as()->value.safeGet(); + structure = checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(arguments[0], context), "structure"); } ColumnsDescription TableFunctionNull::getActualTableStructure(ContextPtr context) const diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 7e7424be38f..d61140e1a07 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -62,9 +62,13 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, Contex throw Exception("Table function 'PostgreSQL' must have arguments.", ErrorCodes::BAD_ARGUMENTS); configuration.emplace(StoragePostgreSQL::getConfiguration(func_args.arguments->children, context)); - connection_pool = std::make_shared(*configuration, - context->getSettingsRef().postgresql_connection_pool_size, - context->getSettingsRef().postgresql_connection_pool_wait_timeout); + const auto & settings = context->getSettingsRef(); + connection_pool = std::make_shared( + *configuration, + settings.postgresql_connection_pool_size, + settings.postgresql_connection_pool_wait_timeout, + POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + settings.postgresql_connection_pool_auto_close_connection); } diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index f06831f191e..098756bcd7c 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include #include @@ -13,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -79,7 +80,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr else { auto database_literal = evaluateConstantExpressionOrIdentifierAsLiteral(arg_value, context); - configuration.database = database_literal->as()->value.safeGet(); + configuration.database = checkAndGetLiteralArgument(database_literal, "database"); } } else @@ -113,7 +114,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr if (is_cluster_function) { args[arg_num] = evaluateConstantExpressionOrIdentifierAsLiteral(args[arg_num], context); - cluster_name = args[arg_num]->as().value.safeGet(); + cluster_name = checkAndGetLiteralArgument(args[arg_num], "cluster_name"); } else { @@ -134,7 +135,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr else { args[arg_num] = evaluateConstantExpressionForDatabaseName(args[arg_num], context); - configuration.database = args[arg_num]->as().value.safeGet(); + configuration.database = checkAndGetLiteralArgument(args[arg_num], "database"); ++arg_num; @@ -149,7 +150,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr { std::swap(qualified_name.database, qualified_name.table); args[arg_num] = evaluateConstantExpressionOrIdentifierAsLiteral(args[arg_num], context); - qualified_name.table = args[arg_num]->as().value.safeGet(); + qualified_name.table = checkAndGetLiteralArgument(args[arg_num], "table"); ++arg_num; } } diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index d081ec4319d..101d946a3f9 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include "registerTableFunctions.h" @@ -56,7 +57,7 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. if (args.size() == 4) { - auto second_arg = args[1]->as().value.safeGet(); + auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); if (FormatFactory::instance().getAllFormats().contains(second_arg)) args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; @@ -68,7 +69,8 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. else if (args.size() == 3) { - auto second_arg = args[1]->as().value.safeGet(); + + auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); if (FormatFactory::instance().getAllFormats().contains(second_arg)) args_to_idx = {{"format", 1}, {"structure", 2}}; else @@ -80,22 +82,22 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar } /// This argument is always the first - s3_configuration.url = args[0]->as().value.safeGet(); + s3_configuration.url = checkAndGetLiteralArgument(args[0], "url"); if (args_to_idx.contains("format")) - s3_configuration.format = args[args_to_idx["format"]]->as().value.safeGet(); + s3_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); if (args_to_idx.contains("structure")) - s3_configuration.structure = args[args_to_idx["structure"]]->as().value.safeGet(); + s3_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); if (args_to_idx.contains("compression_method")) - s3_configuration.compression_method = args[args_to_idx["compression_method"]]->as().value.safeGet(); + s3_configuration.compression_method = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); if (args_to_idx.contains("access_key_id")) - s3_configuration.auth_settings.access_key_id = args[args_to_idx["access_key_id"]]->as().value.safeGet(); + s3_configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); if (args_to_idx.contains("secret_access_key")) - s3_configuration.auth_settings.secret_access_key = args[args_to_idx["secret_access_key"]]->as().value.safeGet(); + s3_configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); } if (s3_configuration.format == "auto") diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index 2f558c58352..fab74c07e11 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -3,11 +3,11 @@ #if USE_AWS_S3 #include +#include +#include #include -#include #include -#include #include #include #include @@ -17,7 +17,6 @@ #include #include #include -#include #include #include "registerTableFunctions.h" @@ -65,7 +64,7 @@ void TableFunctionS3Cluster::parseArguments(const ASTPtr & ast_function, Context throw Exception(message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// This arguments are always the first - configuration.cluster_name = args[0]->as().value.safeGet(); + configuration.cluster_name = checkAndGetLiteralArgument(args[0], "cluster_name"); if (!context->tryGetCluster(configuration.cluster_name)) throw Exception(ErrorCodes::BAD_GET, "Requested cluster '{}' not found", configuration.cluster_name); diff --git a/src/TableFunctions/TableFunctionSQLite.cpp b/src/TableFunctions/TableFunctionSQLite.cpp index fb2dc90a1f7..64ff93494db 100644 --- a/src/TableFunctions/TableFunctionSQLite.cpp +++ b/src/TableFunctions/TableFunctionSQLite.cpp @@ -10,14 +10,14 @@ #include "registerTableFunctions.h" #include -#include #include -#include #include #include +#include + namespace DB { @@ -73,8 +73,8 @@ void TableFunctionSQLite::parseArguments(const ASTPtr & ast_function, ContextPtr for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - database_path = args[0]->as().value.safeGet(); - remote_table_name = args[1]->as().value.safeGet(); + database_path = checkAndGetLiteralArgument(args[0], "database_path"); + remote_table_name = checkAndGetLiteralArgument(args[1], "table_name"); sqlite_db = openSQLiteDB(database_path, context); } diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp index fdc8c4ac911..3baa09a65ea 100644 --- a/src/TableFunctions/TableFunctionZeros.cpp +++ b/src/TableFunctions/TableFunctionZeros.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -55,7 +55,7 @@ void registerTableFunctionZeros(TableFunctionFactory & factory) template UInt64 TableFunctionZeros::evaluateArgument(ContextPtr context, ASTPtr & argument) const { - return evaluateConstantExpressionOrIdentifierAsLiteral(argument, context)->as().value.safeGet(); + return checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(argument, context), "length"); } } diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 45e45b505d4..e84d1645fdf 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -49,6 +49,9 @@ endif() if (TARGET ch_contrib::amqp_cpp) set(USE_AMQPCPP 1) endif() +if (TARGET ch_contrib::nats_io) + set(USE_NATSIO 1) +endif() if (TARGET ch_contrib::cassandra) set(USE_CASSANDRA 1) endif() diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 8d44bace891..a54a8989565 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -6,6 +6,7 @@ import json import logging import subprocess import sys +import time from os import path as p, makedirs from typing import List, Tuple @@ -115,6 +116,34 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def retry_popen(cmd: str) -> int: + max_retries = 5 + for retry in range(max_retries): + # From time to time docker build may failed. Curl issues, or even push + # It will sleep progressively 5, 15, 30 and 50 seconds between retries + progressive_sleep = 5 * sum(i + 1 for i in range(retry)) + if progressive_sleep: + logging.warning( + "The following command failed, sleep %s before retry: %s", + progressive_sleep, + cmd, + ) + time.sleep(progressive_sleep) + with subprocess.Popen( + cmd, + shell=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True, + ) as process: + for line in process.stdout: # type: ignore + print(line, end="") + retcode = process.wait() + if retcode == 0: + return 0 + return retcode + + def auto_release_type(version: ClickHouseVersion, release_type: str) -> str: if release_type != "auto": return release_type @@ -240,41 +269,22 @@ def build_and_push_image( ) cmd = " ".join(cmd_args) logging.info("Building image %s:%s for arch %s: %s", image.repo, tag, arch, cmd) - with subprocess.Popen( - cmd, - shell=True, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - universal_newlines=True, - ) as process: - for line in process.stdout: # type: ignore - print(line, end="") - retcode = process.wait() - if retcode != 0: - result.append((f"{image.repo}:{tag}-{arch}", "FAIL")) - return result - result.append((f"{image.repo}:{tag}-{arch}", "OK")) - with open(metadata_path, "rb") as m: - metadata = json.load(m) - digests.append(metadata["containerimage.digest"]) + if retry_popen(cmd) != 0: + result.append((f"{image.repo}:{tag}-{arch}", "FAIL")) + return result + result.append((f"{image.repo}:{tag}-{arch}", "OK")) + with open(metadata_path, "rb") as m: + metadata = json.load(m) + digests.append(metadata["containerimage.digest"]) if push: cmd = ( "docker buildx imagetools create " f"--tag {image.repo}:{tag} {' '.join(digests)}" ) logging.info("Pushing merged %s:%s image: %s", image.repo, tag, cmd) - with subprocess.Popen( - cmd, - shell=True, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - universal_newlines=True, - ) as process: - for line in process.stdout: # type: ignore - print(line, end="") - retcode = process.wait() - if retcode != 0: - result.append((f"{image.repo}:{tag}", "FAIL")) + if retry_popen(cmd) != 0: + result.append((f"{image.repo}:{tag}", "FAIL")) + return result else: logging.info( "Merging is available only on push, separate %s images are created", diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index a06c5a75b0f..4b7c100c300 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -200,6 +200,13 @@ class PRInfo: ] else: self.diff_urls.append(pull_request["diff_url"]) + if "release" in self.labels: + # For release PRs we must get not only files changed in the PR + # itself, but as well files changed since we branched out + self.diff_urls.append( + f"https://github.com/{GITHUB_REPOSITORY}/" + f"compare/{self.head_ref}...master.diff" + ) else: print("event.json does not match pull_request or push:") print(json.dumps(github_event, sort_keys=True, indent=4)) diff --git a/tests/integration/README.md b/tests/integration/README.md index ef0b5a4b334..2d44ff70861 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -77,25 +77,25 @@ Notes: You can run tests via `./runner` script and pass pytest arguments as last arg: ``` -$ ./runner --binary $HOME/ClickHouse/programs/clickhouse --bridge-binary $HOME/ClickHouse/programs/clickhouse-odbc-bridge --base-configs-dir $HOME/ClickHouse/programs/server/ 'test_odbc_interaction -ss' +$ ./runner --binary $HOME/ClickHouse/programs/clickhouse --odbc-bridge-binary $HOME/ClickHouse/programs/clickhouse-odbc-bridge --base-configs-dir $HOME/ClickHouse/programs/server/ 'test_ssl_cert_authentication -ss' Start tests -============================= test session starts ============================== -platform linux2 -- Python 2.7.15rc1, pytest-4.0.0, py-1.7.0, pluggy-0.8.0 -rootdir: /ClickHouse/tests/integration, inifile: pytest.ini -collected 6 items +====================================================================================================== test session starts ====================================================================================================== +platform linux -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0 -- /usr/bin/python3 +cachedir: .pytest_cache +rootdir: /ClickHouse/tests/integration, configfile: pytest.ini +plugins: repeat-0.9.1, xdist-2.5.0, forked-1.4.0, order-1.0.0, timeout-2.1.0 +timeout: 900.0s +timeout method: signal +timeout func_only: False +collected 4 items -test_odbc_interaction/test.py Removing network clickhouse_default -... +test_ssl_cert_authentication/test.py::test_https Copy common default production configuration from /clickhouse-config. Files: config.xml, users.xml +PASSED +test_ssl_cert_authentication/test.py::test_https_wrong_cert PASSED +test_ssl_cert_authentication/test.py::test_https_non_ssl_auth PASSED +test_ssl_cert_authentication/test.py::test_create_user PASSED -Killing roottestodbcinteraction_node1_1 ... done -Killing roottestodbcinteraction_mysql1_1 ... done -Killing roottestodbcinteraction_postgres1_1 ... done -Removing roottestodbcinteraction_node1_1 ... done -Removing roottestodbcinteraction_mysql1_1 ... done -Removing roottestodbcinteraction_postgres1_1 ... done -Removing network roottestodbcinteraction_default - -==================== 6 passed, 1 warnings in 95.21 seconds ===================== +================================================================================================= 4 passed in 118.58s (0:01:58) ================================================================================================= ``` diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 0d32547358c..5983c886680 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -22,12 +22,14 @@ try: # Please, add modules that required for specific tests only here. # So contributors will be able to run most tests locally # without installing tons of unneeded packages that may be not so easy to install. + import asyncio from cassandra.policies import RoundRobinPolicy import cassandra.cluster import psycopg2 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import pymongo import pymysql + import nats import meilisearch from confluent_kafka.avro.cached_schema_registry_client import ( CachedSchemaRegistryClient, @@ -213,6 +215,11 @@ def check_rabbitmq_is_available(rabbitmq_id): return p.returncode == 0 +async def check_nats_is_available(nats_ip): + nc = await nats.connect("{}:4444".format(nats_ip), user="click", password="house") + return nc.is_connected + + def enable_consistent_hash_plugin(rabbitmq_id): p = subprocess.Popen( ( @@ -336,6 +343,7 @@ class ClickHouseCluster: self.base_kafka_cmd = [] self.base_kerberized_kafka_cmd = [] self.base_rabbitmq_cmd = [] + self.base_nats_cmd = [] self.base_cassandra_cmd = [] self.base_jdbc_bridge_cmd = [] self.base_redis_cmd = [] @@ -352,6 +360,7 @@ class ClickHouseCluster: self.with_kafka = False self.with_kerberized_kafka = False self.with_rabbitmq = False + self.with_nats = False self.with_odbc_drivers = False self.with_hdfs = False self.with_kerberized_hdfs = False @@ -439,6 +448,11 @@ class ClickHouseCluster: self.rabbitmq_dir = p.abspath(p.join(self.instances_dir, "rabbitmq")) self.rabbitmq_logs_dir = os.path.join(self.rabbitmq_dir, "logs") + self.nats_host = "nats1" + self.nats_ip = None + self.nats_port = 4444 + self.nats_docker_id = None + # available when with_nginx == True self.nginx_host = "nginx" self.nginx_ip = None @@ -1012,6 +1026,26 @@ class ClickHouseCluster: ] return self.base_rabbitmq_cmd + def setup_nats_cmd(self, instance, env_variables, docker_compose_yml_dir): + self.with_nats = True + env_variables["NATS_HOST"] = self.nats_host + env_variables["NATS_INTERNAL_PORT"] = "4444" + env_variables["NATS_EXTERNAL_PORT"] = str(self.nats_port) + + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_nats.yml")] + ) + self.base_nats_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_nats.yml"), + ] + return self.base_nats_cmd + def setup_mongo_secure_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mongo = self.with_mongo_secure = True env_variables["MONGO_HOST"] = self.mongo_host @@ -1202,6 +1236,7 @@ class ClickHouseCluster: with_kafka=False, with_kerberized_kafka=False, with_rabbitmq=False, + with_nats=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, @@ -1291,6 +1326,7 @@ class ClickHouseCluster: with_kafka=with_kafka, with_kerberized_kafka=with_kerberized_kafka, with_rabbitmq=with_rabbitmq, + with_nats=with_nats, with_nginx=with_nginx, with_kerberized_hdfs=with_kerberized_hdfs, with_mongo=with_mongo or with_mongo_secure, @@ -1427,6 +1463,11 @@ class ClickHouseCluster: self.setup_rabbitmq_cmd(instance, env_variables, docker_compose_yml_dir) ) + if with_nats and not self.with_nats: + cmds.append( + self.setup_nats_cmd(instance, env_variables, docker_compose_yml_dir) + ) + if with_nginx and not self.with_nginx: cmds.append( self.setup_nginx_cmd(instance, env_variables, docker_compose_yml_dir) @@ -1875,6 +1916,18 @@ class ClickHouseCluster: raise Exception("Cannot wait RabbitMQ container") return False + def wait_nats_is_available(self, nats_ip, max_retries=5): + retries = 0 + while True: + if asyncio.run(check_nats_is_available(nats_ip)): + break + else: + retries += 1 + if retries > max_retries: + raise Exception("NATS is not available") + logging.debug("Waiting for NATS to start up") + time.sleep(1) + def wait_nginx_to_start(self, timeout=60): self.nginx_ip = self.get_instance_ip(self.nginx_host) start = time.time() @@ -2347,6 +2400,14 @@ class ClickHouseCluster: if self.wait_rabbitmq_to_start(throw=(i == 4)): break + if self.with_nats and self.base_nats_cmd: + logging.debug("Setup NATS") + subprocess_check_call(self.base_nats_cmd + common_opts) + self.nats_docker_id = self.get_instance_docker_id("nats1") + self.up_called = True + self.nats_ip = self.get_instance_ip("nats1") + self.wait_nats_is_available(self.nats_ip) + if self.with_hdfs and self.base_hdfs_cmd: logging.debug("Setup HDFS") os.makedirs(self.hdfs_logs_dir) @@ -2708,6 +2769,7 @@ class ClickHouseInstance: with_kafka, with_kerberized_kafka, with_rabbitmq, + with_nats, with_nginx, with_kerberized_hdfs, with_mongo, @@ -2789,6 +2851,7 @@ class ClickHouseInstance: self.with_kafka = with_kafka self.with_kerberized_kafka = with_kerberized_kafka self.with_rabbitmq = with_rabbitmq + self.with_nats = with_nats self.with_nginx = with_nginx self.with_kerberized_hdfs = with_kerberized_hdfs self.with_mongo = with_mongo @@ -3771,6 +3834,9 @@ class ClickHouseInstance: if self.with_rabbitmq: depends_on.append("rabbitmq1") + if self.with_nats: + depends_on.append("nats1") + if self.with_zookeeper: depends_on.append("zoo1") depends_on.append("zoo2") diff --git a/tests/integration/test_storage_nats/__init__.py b/tests/integration/test_storage_nats/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto b/tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto new file mode 100755 index 00000000000..090ed917cdd --- /dev/null +++ b/tests/integration/test_storage_nats/clickhouse_path/format_schemas/nats.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + + message ProtoKeyValue { + uint64 key = 1; + string value = 2; + } diff --git a/tests/integration/test_storage_nats/configs/macros.xml b/tests/integration/test_storage_nats/configs/macros.xml new file mode 100644 index 00000000000..4aa547e049e --- /dev/null +++ b/tests/integration/test_storage_nats/configs/macros.xml @@ -0,0 +1,7 @@ + + + nats1:4444 + macro + JSONEachRow + + diff --git a/tests/integration/test_storage_nats/configs/named_collection.xml b/tests/integration/test_storage_nats/configs/named_collection.xml new file mode 100644 index 00000000000..15817f321f0 --- /dev/null +++ b/tests/integration/test_storage_nats/configs/named_collection.xml @@ -0,0 +1,13 @@ + + + + nats1:4444 + named + JSONEachRow + 111 + 12 + click + house + + + diff --git a/tests/integration/test_storage_nats/configs/nats.xml b/tests/integration/test_storage_nats/configs/nats.xml new file mode 100644 index 00000000000..0a8be9122ad --- /dev/null +++ b/tests/integration/test_storage_nats/configs/nats.xml @@ -0,0 +1,6 @@ + + + click + house + + diff --git a/tests/integration/test_storage_nats/configs/users.xml b/tests/integration/test_storage_nats/configs/users.xml new file mode 100644 index 00000000000..797dfebba0e --- /dev/null +++ b/tests/integration/test_storage_nats/configs/users.xml @@ -0,0 +1,8 @@ + + + + + 1 + + + diff --git a/tests/integration/test_storage_nats/nats_pb2.py b/tests/integration/test_storage_nats/nats_pb2.py new file mode 100644 index 00000000000..4330ff57950 --- /dev/null +++ b/tests/integration/test_storage_nats/nats_pb2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: clickhouse_path/format_schemas/nats.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n)clickhouse_path/format_schemas/nats.proto"+\n\rProtoKeyValue\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3' +) + + +_PROTOKEYVALUE = DESCRIPTOR.message_types_by_name["ProtoKeyValue"] +ProtoKeyValue = _reflection.GeneratedProtocolMessageType( + "ProtoKeyValue", + (_message.Message,), + { + "DESCRIPTOR": _PROTOKEYVALUE, + "__module__": "clickhouse_path.format_schemas.nats_pb2" + # @@protoc_insertion_point(class_scope:ProtoKeyValue) + }, +) +_sym_db.RegisterMessage(ProtoKeyValue) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _PROTOKEYVALUE._serialized_start = 45 + _PROTOKEYVALUE._serialized_end = 88 +# @@protoc_insertion_point(module_scope) diff --git a/tests/integration/test_storage_nats/test.py b/tests/integration/test_storage_nats/test.py new file mode 100644 index 00000000000..a952f4b78a6 --- /dev/null +++ b/tests/integration/test_storage_nats/test.py @@ -0,0 +1,1494 @@ +import json +import os.path as p +import random +import subprocess +import threading +import logging +import time +from random import randrange +import math + +import asyncio +import nats +import pytest +from google.protobuf.internal.encoder import _VarintBytes +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster, check_nats_is_available +from helpers.test_tools import TSV + +from . import nats_pb2 + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "instance", + main_configs=[ + "configs/nats.xml", + "configs/macros.xml", + "configs/named_collection.xml", + ], + user_configs=["configs/users.xml"], + with_nats=True, + clickhouse_path_dir="clickhouse_path", +) + + +# Helpers + + +def wait_nats_to_start(nats_ip, timeout=180): + start = time.time() + while time.time() - start < timeout: + try: + if asyncio.run(check_nats_is_available(nats_ip)): + logging.debug("NATS is available") + return + time.sleep(0.5) + except Exception as ex: + logging.debug("Can't connect to NATS " + str(ex)) + time.sleep(0.5) + + +def nats_check_result(result, check=False, ref_file="test_nats_json.reference"): + fpath = p.join(p.dirname(__file__), ref_file) + with open(fpath) as reference: + if check: + assert TSV(result) == TSV(reference) + else: + return TSV(result) == TSV(reference) + + +def kill_nats(nats_id): + p = subprocess.Popen(("docker", "stop", nats_id), stdout=subprocess.PIPE) + p.communicate() + return p.returncode == 0 + + +def revive_nats(nats_id, nats_ip): + p = subprocess.Popen(("docker", "start", nats_id), stdout=subprocess.PIPE) + p.communicate() + wait_nats_to_start(nats_ip) + + +# Fixtures + + +@pytest.fixture(scope="module") +def nats_cluster(): + try: + cluster.start() + logging.debug("nats_id is {}".format(instance.cluster.nats_docker_id)) + instance.query("CREATE DATABASE test") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def nats_setup_teardown(): + print("NATS is available - running test") + yield # run test + instance.query("DROP DATABASE test NO DELAY") + instance.query("CREATE DATABASE test") + + +# Tests + + +async def nats_produce_messages(ip, subject, messages=(), bytes=None): + nc = await nats.connect("{}:4444".format(ip), user="click", password="house") + logging.debug("NATS connection status: " + str(nc.is_connected)) + + for message in messages: + await nc.publish(subject, message.encode()) + if bytes is not None: + await nc.publish(subject, bytes) + logging.debug("Finished publising to " + subject) + + await nc.close() + return messages + + +def check_table_is_ready(instance, table_name): + try: + instance.query("SELECT * FROM {}".format(table_name)) + return True + except Exception: + return False + + +def test_nats_select(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'select', + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(50): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "select", messages)) + + # The order of messages in select * from test.nats is not guaranteed, so sleep to collect everything in one select + time.sleep(1) + + result = "" + while True: + result += instance.query( + "SELECT * FROM test.nats ORDER BY key", ignore_error=True + ) + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_select_empty(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'empty', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + """ + ) + + assert int(instance.query("SELECT count() FROM test.nats")) == 0 + + +def test_nats_json_without_delimiter(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'json', + nats_format = 'JSONEachRow'; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = "" + for i in range(25): + messages += json.dumps({"key": i, "value": i}) + "\n" + + all_messages = [messages] + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "json", all_messages)) + + messages = "" + for i in range(25, 50): + messages += json.dumps({"key": i, "value": i}) + "\n" + all_messages = [messages] + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "json", all_messages)) + + time.sleep(1) + + result = "" + time_limit_sec = 60 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result += instance.query( + "SELECT * FROM test.nats ORDER BY key", ignore_error=True + ) + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_csv_with_delimiter(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'csv', + nats_format = 'CSV', + nats_row_delimiter = '\\n'; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(50): + messages.append("{i}, {i}".format(i=i)) + + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "csv", messages)) + + time.sleep(1) + + result = "" + for _ in range(60): + result += instance.query( + "SELECT * FROM test.nats ORDER BY key", ignore_error=True + ) + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_tsv_with_delimiter(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'tsv', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.nats; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(50): + messages.append("{i}\t{i}".format(i=i)) + + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "tsv", messages)) + + result = "" + for _ in range(60): + result = instance.query("SELECT * FROM test.view ORDER BY key") + if nats_check_result(result): + break + + nats_check_result(result, True) + + +# + + +def test_nats_macros(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = '{nats_url}', + nats_subjects = '{nats_subjects}', + nats_format = '{nats_format}' + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + message = "" + for i in range(50): + message += json.dumps({"key": i, "value": i}) + "\n" + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "macro", [message])) + + time.sleep(1) + + result = "" + for _ in range(60): + result += instance.query( + "SELECT * FROM test.nats ORDER BY key", ignore_error=True + ) + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_materialized_view(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'mv', + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.nats; + + CREATE TABLE test.view2 (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS + SELECT * FROM test.nats group by (key, value); + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(50): + messages.append(json.dumps({"key": i, "value": i})) + + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages)) + + time_limit_sec = 60 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result = instance.query("SELECT * FROM test.view ORDER BY key") + if nats_check_result(result): + break + + nats_check_result(result, True) + + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result = instance.query("SELECT * FROM test.view2 ORDER BY key") + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_materialized_view_with_subquery(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'mvsq', + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM (SELECT * FROM test.nats); + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(50): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mvsq", messages)) + + time_limit_sec = 60 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result = instance.query("SELECT * FROM test.view ORDER BY key") + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_many_materialized_views(nats_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.view1; + DROP TABLE IF EXISTS test.view2; + DROP TABLE IF EXISTS test.consumer1; + DROP TABLE IF EXISTS test.consumer2; + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'mmv', + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view1 (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE TABLE test.view2 (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer1 TO test.view1 AS + SELECT * FROM test.nats; + CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS + SELECT * FROM test.nats; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(50): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mmv", messages)) + + time_limit_sec = 60 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result1 = instance.query("SELECT * FROM test.view1 ORDER BY key") + result2 = instance.query("SELECT * FROM test.view2 ORDER BY key") + if nats_check_result(result1) and nats_check_result(result2): + break + + instance.query( + """ + DROP TABLE test.consumer1; + DROP TABLE test.consumer2; + DROP TABLE test.view1; + DROP TABLE test.view2; + """ + ) + + nats_check_result(result1, True) + nats_check_result(result2, True) + + +def test_nats_protobuf(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value String) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'pb', + nats_format = 'Protobuf', + nats_schema = 'nats.proto:ProtoKeyValue'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.nats; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + data = b"" + for i in range(0, 20): + msg = nats_pb2.ProtoKeyValue() + msg.key = i + msg.value = str(i) + serialized_msg = msg.SerializeToString() + data = data + _VarintBytes(len(serialized_msg)) + serialized_msg + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "pb", bytes=data)) + data = b"" + for i in range(20, 21): + msg = nats_pb2.ProtoKeyValue() + msg.key = i + msg.value = str(i) + serialized_msg = msg.SerializeToString() + data = data + _VarintBytes(len(serialized_msg)) + serialized_msg + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "pb", bytes=data)) + data = b"" + for i in range(21, 50): + msg = nats_pb2.ProtoKeyValue() + msg.key = i + msg.value = str(i) + serialized_msg = msg.SerializeToString() + data = data + _VarintBytes(len(serialized_msg)) + serialized_msg + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "pb", bytes=data)) + + result = "" + time_limit_sec = 60 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result = instance.query("SELECT * FROM test.view ORDER BY key") + if nats_check_result(result): + break + + nats_check_result(result, True) + + +def test_nats_big_message(nats_cluster): + # Create batchs of messages of size ~100Kb + nats_messages = 1000 + batch_messages = 1000 + messages = [ + json.dumps({"key": i, "value": "x" * 100}) * batch_messages + for i in range(nats_messages) + ] + + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value String) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'big', + nats_format = 'JSONEachRow'; + CREATE TABLE test.view (key UInt64, value String) + ENGINE = MergeTree + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.nats; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "big", messages)) + + while True: + result = instance.query("SELECT count() FROM test.view") + if int(result) == batch_messages * nats_messages: + break + + assert ( + int(result) == nats_messages * batch_messages + ), "ClickHouse lost some messages: {}".format(result) + + +def test_nats_mv_combo(nats_cluster): + NUM_MV = 5 + NUM_CONSUMERS = 4 + + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'combo', + nats_num_consumers = {}, + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + """.format( + NUM_CONSUMERS + ) + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + for mv_id in range(NUM_MV): + instance.query( + """ + DROP TABLE IF EXISTS test.combo_{0}; + DROP TABLE IF EXISTS test.combo_{0}_mv; + CREATE TABLE test.combo_{0} (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.combo_{0}_mv TO test.combo_{0} AS + SELECT * FROM test.nats; + """.format( + mv_id + ) + ) + + time.sleep(2) + + i = [0] + messages_num = 10000 + + def produce(): + messages = [] + for _ in range(messages_num): + messages.append(json.dumps({"key": i[0], "value": i[0]})) + i[0] += 1 + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "combo", messages)) + + threads = [] + threads_num = 20 + + for _ in range(threads_num): + threads.append(threading.Thread(target=produce)) + for thread in threads: + time.sleep(random.uniform(0, 1)) + thread.start() + + while True: + result = 0 + for mv_id in range(NUM_MV): + result += int( + instance.query("SELECT count() FROM test.combo_{0}".format(mv_id)) + ) + if int(result) == messages_num * threads_num * NUM_MV: + break + time.sleep(1) + + for thread in threads: + thread.join() + + for mv_id in range(NUM_MV): + instance.query( + """ + DROP TABLE test.combo_{0}_mv; + DROP TABLE test.combo_{0}; + """.format( + mv_id + ) + ) + + assert ( + int(result) == messages_num * threads_num * NUM_MV + ), "ClickHouse lost some messages: {}".format(result) + + +def test_nats_insert(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'insert', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + values = [] + for i in range(50): + values.append("({i}, {i})".format(i=i)) + values = ",".join(values) + + insert_messages = [] + + async def sub_to_nats(): + nc = await nats.connect( + "{}:4444".format(nats_cluster.nats_ip), user="click", password="house" + ) + sub = await nc.subscribe("insert") + await sub.unsubscribe(50) + async for msg in sub.messages: + insert_messages.append(msg.data.decode()) + + await sub.drain() + await nc.drain() + + def run_sub(): + asyncio.run(sub_to_nats()) + + thread = threading.Thread(target=run_sub) + thread.start() + time.sleep(1) + + while True: + try: + instance.query("INSERT INTO test.nats VALUES {}".format(values)) + break + except QueryRuntimeException as e: + if "Local: Timed out." in str(e): + continue + else: + raise + thread.join() + + result = "\n".join(insert_messages) + nats_check_result(result, True) + + +def test_nats_many_subjects_insert_wrong(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'insert1,insert2.>,insert3.*.foo', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + values = [] + for i in range(50): + values.append("({i}, {i})".format(i=i)) + values = ",".join(values) + + # no subject specified + instance.query_and_get_error("INSERT INTO test.nats VALUES {}".format(values)) + + # can't insert into wildcard subjects + instance.query_and_get_error( + "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert2.>' VALUES {}".format( + values + ) + ) + instance.query_and_get_error( + "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert3.*.foo' VALUES {}".format( + values + ) + ) + + # specified subject is not among engine's subjects + instance.query_and_get_error( + "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert4' VALUES {}".format( + values + ) + ) + instance.query_and_get_error( + "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='insert3.foo.baz' VALUES {}".format( + values + ) + ) + instance.query_and_get_error( + "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='foo.insert2' VALUES {}".format( + values + ) + ) + + +def test_nats_many_subjects_insert_right(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'right_insert1,right_insert2', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + values = [] + for i in range(50): + values.append("({i}, {i})".format(i=i)) + values = ",".join(values) + + insert_messages = [] + + async def sub_to_nats(): + nc = await nats.connect( + "{}:4444".format(nats_cluster.nats_ip), user="click", password="house" + ) + sub = await nc.subscribe("right_insert1") + await sub.unsubscribe(50) + async for msg in sub.messages: + insert_messages.append(msg.data.decode()) + + await sub.drain() + await nc.drain() + + def run_sub(): + asyncio.run(sub_to_nats()) + + thread = threading.Thread(target=run_sub) + thread.start() + time.sleep(1) + + while True: + try: + instance.query( + "INSERT INTO test.nats SETTINGS stream_like_engine_insert_queue='right_insert1' VALUES {}".format( + values + ) + ) + break + except QueryRuntimeException as e: + if "Local: Timed out." in str(e): + continue + else: + raise + thread.join() + + result = "\n".join(insert_messages) + nats_check_result(result, True) + + +def test_nats_many_inserts(nats_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.nats_many; + DROP TABLE IF EXISTS test.nats_consume; + DROP TABLE IF EXISTS test.view_many; + DROP TABLE IF EXISTS test.consumer_many; + CREATE TABLE test.nats_many (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'many_inserts', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + CREATE TABLE test.nats_consume (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'many_inserts', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view_many (key UInt64, value UInt64) + ENGINE = MergeTree + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer_many TO test.view_many AS + SELECT * FROM test.nats_consume; + """ + ) + while not check_table_is_ready(instance, "test.nats_consume"): + logging.debug("Table test.nats_consume is not yet ready") + time.sleep(0.5) + + messages_num = 10000 + values = [] + for i in range(messages_num): + values.append("({i}, {i})".format(i=i)) + values = ",".join(values) + + def insert(): + while True: + try: + instance.query("INSERT INTO test.nats_many VALUES {}".format(values)) + break + except QueryRuntimeException as e: + if "Local: Timed out." in str(e): + continue + else: + raise + + threads = [] + threads_num = 10 + for _ in range(threads_num): + threads.append(threading.Thread(target=insert)) + for thread in threads: + time.sleep(random.uniform(0, 1)) + thread.start() + + for thread in threads: + thread.join() + + time_limit_sec = 300 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result = instance.query("SELECT count() FROM test.view_many") + print(result, messages_num * threads_num) + if int(result) >= messages_num * threads_num: + break + time.sleep(1) + + instance.query( + """ + DROP TABLE test.nats_consume; + DROP TABLE test.nats_many; + DROP TABLE test.consumer_many; + DROP TABLE test.view_many; + """ + ) + + assert ( + int(result) == messages_num * threads_num + ), "ClickHouse lost some messages or got duplicated ones. Total count: {}".format( + result + ) + + +def test_nats_overloaded_insert(nats_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.view_overload; + DROP TABLE IF EXISTS test.consumer_overload; + DROP TABLE IF EXISTS test.nats_consume; + CREATE TABLE test.nats_consume (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'over', + nats_num_consumers = 5, + nats_max_block_size = 10000, + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + CREATE TABLE test.nats_overload (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'over', + nats_format = 'TSV', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view_overload (key UInt64, value UInt64) + ENGINE = MergeTree + ORDER BY key + SETTINGS old_parts_lifetime=5, cleanup_delay_period=2, cleanup_delay_period_random_add=3; + CREATE MATERIALIZED VIEW test.consumer_overload TO test.view_overload AS + SELECT * FROM test.nats_consume; + """ + ) + while not check_table_is_ready(instance, "test.nats_consume"): + logging.debug("Table test.nats_consume is not yet ready") + time.sleep(0.5) + + messages_num = 100000 + + def insert(): + values = [] + for i in range(messages_num): + values.append("({i}, {i})".format(i=i)) + values = ",".join(values) + + while True: + try: + instance.query( + "INSERT INTO test.nats_overload VALUES {}".format(values) + ) + break + except QueryRuntimeException as e: + if "Local: Timed out." in str(e): + continue + else: + raise + + threads = [] + threads_num = 5 + for _ in range(threads_num): + threads.append(threading.Thread(target=insert)) + for thread in threads: + time.sleep(random.uniform(0, 1)) + thread.start() + + time_limit_sec = 300 + deadline = time.monotonic() + time_limit_sec + + while time.monotonic() < deadline: + result = instance.query("SELECT count() FROM test.view_overload") + time.sleep(1) + if int(result) >= messages_num * threads_num: + break + + instance.query( + """ + DROP TABLE test.consumer_overload; + DROP TABLE test.view_overload; + DROP TABLE test.nats_consume; + DROP TABLE test.nats_overload; + """ + ) + + for thread in threads: + thread.join() + + assert ( + int(result) == messages_num * threads_num + ), "ClickHouse lost some messages or got duplicated ones. Total count: {}".format( + result + ) + + +def test_nats_virtual_column(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats_virtuals (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'virtuals', + nats_format = 'JSONEachRow'; + CREATE MATERIALIZED VIEW test.view Engine=Log AS + SELECT value, key, _subject FROM test.nats_virtuals; + """ + ) + while not check_table_is_ready(instance, "test.nats_virtuals"): + logging.debug("Table test.nats_virtuals is not yet ready") + time.sleep(0.5) + + message_num = 10 + i = 0 + messages = [] + for _ in range(message_num): + messages.append(json.dumps({"key": i, "value": i})) + i += 1 + + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "virtuals", messages)) + + while True: + result = instance.query("SELECT count() FROM test.view") + time.sleep(1) + if int(result) == message_num: + break + + result = instance.query( + """ + SELECT key, value, _subject + FROM test.view ORDER BY key + """ + ) + + expected = """\ +0 0 virtuals +1 1 virtuals +2 2 virtuals +3 3 virtuals +4 4 virtuals +5 5 virtuals +6 6 virtuals +7 7 virtuals +8 8 virtuals +9 9 virtuals +""" + + instance.query( + """ + DROP TABLE test.nats_virtuals; + DROP TABLE test.view; + """ + ) + + assert TSV(result) == TSV(expected) + + +def test_nats_virtual_column_with_materialized_view(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats_virtuals_mv (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'virtuals_mv', + nats_format = 'JSONEachRow'; + CREATE TABLE test.view (key UInt64, value UInt64, subject String) ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT *, _subject as subject + FROM test.nats_virtuals_mv; + """ + ) + while not check_table_is_ready(instance, "test.nats_virtuals_mv"): + logging.debug("Table test.nats_virtuals_mv is not yet ready") + time.sleep(0.5) + + message_num = 10 + i = 0 + messages = [] + for _ in range(message_num): + messages.append(json.dumps({"key": i, "value": i})) + i += 1 + + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "virtuals_mv", messages)) + + while True: + result = instance.query("SELECT count() FROM test.view") + time.sleep(1) + if int(result) == message_num: + break + + result = instance.query("SELECT key, value, subject FROM test.view ORDER BY key") + expected = """\ +0 0 virtuals_mv +1 1 virtuals_mv +2 2 virtuals_mv +3 3 virtuals_mv +4 4 virtuals_mv +5 5 virtuals_mv +6 6 virtuals_mv +7 7 virtuals_mv +8 8 virtuals_mv +9 9 virtuals_mv +""" + + instance.query( + """ + DROP TABLE test.consumer; + DROP TABLE test.view; + DROP TABLE test.nats_virtuals_mv + """ + ) + + assert TSV(result) == TSV(expected) + + +def test_nats_many_consumers_to_each_queue(nats_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.destination; + CREATE TABLE test.destination(key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + """ + ) + + num_tables = 4 + for table_id in range(num_tables): + print(("Setting up table {}".format(table_id))) + instance.query( + """ + DROP TABLE IF EXISTS test.many_consumers_{0}; + DROP TABLE IF EXISTS test.many_consumers_{0}_mv; + CREATE TABLE test.many_consumers_{0} (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'many_consumers', + nats_num_consumers = 2, + nats_queue_group = 'many_consumers', + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + CREATE MATERIALIZED VIEW test.many_consumers_{0}_mv TO test.destination AS + SELECT key, value FROM test.many_consumers_{0}; + """.format( + table_id + ) + ) + while not check_table_is_ready( + instance, "test.many_consumers_{}".format(table_id) + ): + logging.debug( + "Table test.many_consumers_{} is not yet ready".format(table_id) + ) + time.sleep(0.5) + + i = [0] + messages_num = 1000 + + def produce(): + messages = [] + for _ in range(messages_num): + messages.append(json.dumps({"key": i[0], "value": i[0]})) + i[0] += 1 + asyncio.run( + nats_produce_messages(nats_cluster.nats_ip, "many_consumers", messages) + ) + + threads = [] + threads_num = 20 + + for _ in range(threads_num): + threads.append(threading.Thread(target=produce)) + for thread in threads: + time.sleep(random.uniform(0, 1)) + thread.start() + + result1 = "" + while True: + result1 = instance.query("SELECT count() FROM test.destination") + time.sleep(1) + if int(result1) == messages_num * threads_num: + break + + for thread in threads: + thread.join() + + for consumer_id in range(num_tables): + instance.query( + """ + DROP TABLE test.many_consumers_{0}; + DROP TABLE test.many_consumers_{0}_mv; + """.format( + consumer_id + ) + ) + + instance.query( + """ + DROP TABLE test.destination; + """ + ) + + assert ( + int(result1) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result1) + + +def test_nats_restore_failed_connection_without_losses_on_write(nats_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.consume; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree + ORDER BY key; + CREATE TABLE test.consume (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'producer_reconnect', + nats_format = 'JSONEachRow', + nats_num_consumers = 2, + nats_row_delimiter = '\\n'; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.consume; + DROP TABLE IF EXISTS test.producer_reconnect; + CREATE TABLE test.producer_reconnect (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'producer_reconnect', + nats_format = 'JSONEachRow', + nats_row_delimiter = '\\n'; + """ + ) + while not check_table_is_ready(instance, "test.consume"): + logging.debug("Table test.consume is not yet ready") + time.sleep(0.5) + + messages_num = 100000 + values = [] + for i in range(messages_num): + values.append("({i}, {i})".format(i=i)) + values = ",".join(values) + + while True: + try: + instance.query( + "INSERT INTO test.producer_reconnect VALUES {}".format(values) + ) + break + except QueryRuntimeException as e: + if "Local: Timed out." in str(e): + continue + else: + raise + + while int(instance.query("SELECT count() FROM test.view")) == 0: + time.sleep(0.1) + + kill_nats(nats_cluster.nats_docker_id) + time.sleep(4) + revive_nats(nats_cluster.nats_docker_id, nats_cluster.nats_ip) + + while True: + result = instance.query("SELECT count(DISTINCT key) FROM test.view") + time.sleep(1) + if int(result) == messages_num: + break + + instance.query( + """ + DROP TABLE test.consume; + DROP TABLE test.producer_reconnect; + """ + ) + + assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( + result + ) + + +def test_nats_no_connection_at_startup_1(nats_cluster): + # no connection when table is initialized + nats_cluster.pause_container("nats1") + instance.query_and_get_error( + """ + CREATE TABLE test.cs (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'cs', + nats_format = 'JSONEachRow', + nats_num_consumers = '5', + nats_row_delimiter = '\\n'; + """ + ) + nats_cluster.unpause_container("nats1") + + +def test_nats_no_connection_at_startup_2(nats_cluster): + instance.query( + """ + CREATE TABLE test.cs (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'cs', + nats_format = 'JSONEachRow', + nats_num_consumers = '5', + nats_row_delimiter = '\\n'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.cs; + """ + ) + + instance.query("DETACH TABLE test.cs") + nats_cluster.pause_container("nats1") + instance.query("ATTACH TABLE test.cs") + nats_cluster.unpause_container("nats1") + while not check_table_is_ready(instance, "test.cs"): + logging.debug("Table test.cs is not yet ready") + time.sleep(0.5) + + messages_num = 1000 + messages = [] + for i in range(messages_num): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "cs", messages)) + + for _ in range(20): + result = instance.query("SELECT count() FROM test.view") + time.sleep(1) + if int(result) == messages_num: + break + + instance.query( + """ + DROP TABLE test.consumer; + DROP TABLE test.cs; + """ + ) + + assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( + result + ) + + +def test_nats_format_factory_settings(nats_cluster): + instance.query( + """ + CREATE TABLE test.format_settings ( + id String, date DateTime + ) ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'format_settings', + nats_format = 'JSONEachRow', + date_time_input_format = 'best_effort'; + """ + ) + while not check_table_is_ready(instance, "test.format_settings"): + logging.debug("Table test.format_settings is not yet ready") + time.sleep(0.5) + + message = json.dumps( + {"id": "format_settings_test", "date": "2021-01-19T14:42:33.1829214Z"} + ) + expected = instance.query( + """SELECT parseDateTimeBestEffort(CAST('2021-01-19T14:42:33.1829214Z', 'String'))""" + ) + + asyncio.run( + nats_produce_messages(nats_cluster.nats_ip, "format_settings", [message]) + ) + + while True: + result = instance.query("SELECT date FROM test.format_settings") + if result == expected: + break + + instance.query( + """ + CREATE TABLE test.view ( + id String, date DateTime + ) ENGINE = MergeTree ORDER BY id; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.format_settings; + """ + ) + + asyncio.run( + nats_produce_messages(nats_cluster.nats_ip, "format_settings", [message]) + ) + while True: + result = instance.query("SELECT date FROM test.view") + if result == expected: + break + + instance.query( + """ + DROP TABLE test.consumer; + DROP TABLE test.format_settings; + """ + ) + + assert result == expected + + +def test_nats_bad_args(nats_cluster): + instance.query_and_get_error( + """ + CREATE TABLE test.drop (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_secure = true, + nats_format = 'JSONEachRow'; + """ + ) + + +def test_nats_drop_mv(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = 'nats1:4444', + nats_subjects = 'mv', + nats_format = 'JSONEachRow'; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.nats; + """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + messages = [] + for i in range(20): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages)) + + instance.query("DROP VIEW test.consumer") + messages = [] + for i in range(20, 40): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages)) + + instance.query( + """ + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.nats; + """ + ) + messages = [] + for i in range(40, 50): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages)) + + while True: + result = instance.query("SELECT * FROM test.view ORDER BY key") + if nats_check_result(result): + break + + nats_check_result(result, True) + + instance.query("DROP VIEW test.consumer") + messages = [] + for i in range(50, 60): + messages.append(json.dumps({"key": i, "value": i})) + asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages)) + + count = 0 + while True: + count = int(instance.query("SELECT count() FROM test.nats")) + if count: + break + + assert count > 0 + + +def test_nats_predefined_configuration(nats_cluster): + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS(nats1) """ + ) + while not check_table_is_ready(instance, "test.nats"): + logging.debug("Table test.nats is not yet ready") + time.sleep(0.5) + + asyncio.run( + nats_produce_messages( + nats_cluster.nats_ip, "named", [json.dumps({"key": 1, "value": 2})] + ) + ) + while True: + result = instance.query( + "SELECT * FROM test.nats ORDER BY key", ignore_error=True + ) + if result == "1\t2\n": + break + + +if __name__ == "__main__": + cluster.start() + input("Cluster created, press any key to destroy...") + cluster.shutdown() diff --git a/tests/integration/test_storage_nats/test_nats_json.reference b/tests/integration/test_storage_nats/test_nats_json.reference new file mode 100644 index 00000000000..959bb2aad74 --- /dev/null +++ b/tests/integration/test_storage_nats/test_nats_json.reference @@ -0,0 +1,50 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +20 20 +21 21 +22 22 +23 23 +24 24 +25 25 +26 26 +27 27 +28 28 +29 29 +30 30 +31 31 +32 32 +33 33 +34 34 +35 35 +36 36 +37 37 +38 38 +39 39 +40 40 +41 41 +42 42 +43 43 +44 44 +45 45 +46 46 +47 47 +48 48 +49 49 diff --git a/tests/integration/test_storage_postgresql/configs/settings.xml b/tests/integration/test_storage_postgresql/configs/settings.xml new file mode 100644 index 00000000000..7054c274771 --- /dev/null +++ b/tests/integration/test_storage_postgresql/configs/settings.xml @@ -0,0 +1,8 @@ + + + + + 1 + + + diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 1fc0475419c..a3ebbe97451 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -10,7 +10,10 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/named_collections.xml"], with_postgres=True ) node2 = cluster.add_instance( - "node2", main_configs=["configs/named_collections.xml"], with_postgres_cluster=True + "node2", + main_configs=["configs/named_collections.xml"], + user_configs=["configs/settings.xml"], + with_postgres_cluster=True, ) @@ -19,6 +22,7 @@ def started_cluster(): try: cluster.start() node1.query("CREATE DATABASE test") + node2.query("CREATE DATABASE test") yield cluster finally: @@ -640,6 +644,55 @@ def test_uuid(started_cluster): assert result.strip() == "Nullable(UUID)" +def test_auto_close_connection(started_cluster): + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=False + ) + cursor = conn.cursor() + database_name = "auto_close_connection_test" + + cursor.execute(f"DROP DATABASE IF EXISTS {database_name}") + cursor.execute(f"CREATE DATABASE {database_name}") + conn = get_postgres_conn( + started_cluster.postgres_ip, + started_cluster.postgres_port, + database=True, + database_name=database_name, + ) + cursor = conn.cursor() + cursor.execute("CREATE TABLE test_table (key integer, value integer)") + + node2.query( + f""" + CREATE TABLE test.test_table (key UInt32, value UInt32) + ENGINE = PostgreSQL(postgres1, database='{database_name}', table='test_table') + """ + ) + + result = node2.query( + "INSERT INTO test.test_table SELECT number, number FROM numbers(1000)", + user="default", + ) + + result = node2.query("SELECT * FROM test.test_table LIMIT 100", user="default") + + node2.query( + f""" + CREATE TABLE test.stat (numbackends UInt32, datname String) + ENGINE = PostgreSQL(postgres1, database='{database_name}', table='pg_stat_database') + """ + ) + + count = int( + node2.query( + f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'" + ) + ) + + # Connection from python + pg_stat table also has a connection at the moment of current query + assert count == 2 + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 64f104dd4ff..18b1e9d974b 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -30,6 +30,11 @@ instance = cluster.add_instance( stay_alive=True, ) +instance2 = cluster.add_instance( + "instance2", + user_configs=["configs/users.xml"], + with_rabbitmq=True, +) # Helpers @@ -2782,3 +2787,47 @@ def test_rabbitmq_msgpack(rabbitmq_cluster): break time.sleep(1) assert result.strip() == "kek" + + instance.query("drop table rabbit_in sync") + instance.query("drop table rabbit_out sync") + + +def test_rabbitmq_address(rabbitmq_cluster): + + instance2.query( + """ + drop table if exists rabbit_in; + drop table if exists rabbit_out; + create table + rabbit_in (val String) + engine=RabbitMQ + SETTINGS rabbitmq_exchange_name = 'rxhep', + rabbitmq_format = 'CSV', + rabbitmq_num_consumers = 1, + rabbitmq_address='amqp://root:clickhouse@rabbitmq1:5672/'; + create table + rabbit_out (val String) engine=RabbitMQ + SETTINGS rabbitmq_exchange_name = 'rxhep', + rabbitmq_format = 'CSV', + rabbitmq_num_consumers = 1, + rabbitmq_address='amqp://root:clickhouse@rabbitmq1:5672/'; + set stream_like_engine_allow_direct_select=1; + insert into rabbit_out select 'kek'; + """ + ) + + result = "" + try_no = 0 + while True: + result = instance2.query("select * from rabbit_in;") + if result.strip() == "kek": + break + else: + try_no = try_no + 1 + if try_no == 20: + break + time.sleep(1) + assert result.strip() == "kek" + + instance2.query("drop table rabbit_in sync") + instance2.query("drop table rabbit_out sync") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index ec7c746c549..5dd09ddd362 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -1052,61 +1052,61 @@ def test_seekable_formats(started_cluster): instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" - instance.query( - f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1" + exec_query_with_retry( + instance, + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", ) result = instance.query(f"SELECT count() FROM {table_function}") - assert int(result) == 5000000 + assert int(result) == 1000000 table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" exec_query_with_retry( instance, - f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1", + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", ) - result = instance.query(f"SELECT count() FROM {table_function}") - assert int(result) == 5000000 + result = instance.query( + f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='50M'" + ) + assert int(result) == 1000000 + + instance.query(f"SELECT * FROM {table_function} FORMAT Null") instance.query("SYSTEM FLUSH LOGS") result = instance.query( - f"SELECT formatReadableSize(memory_usage) FROM system.query_log WHERE startsWith(query, 'SELECT count() FROM s3') AND memory_usage > 0 ORDER BY event_time desc" + f"SELECT formatReadableSize(ProfileEvents['ReadBufferFromS3Bytes']) FROM system.query_log WHERE startsWith(query, 'SELECT * FROM s3') AND memory_usage > 0 AND type='QueryFinish' ORDER BY event_time_microseconds DESC LIMIT 1" ) - + result = result.strip() + assert result.endswith("MiB") result = result[: result.index(".")] - assert int(result) < 200 + assert int(result) > 80 def test_seekable_formats_url(started_cluster): bucket = started_cluster.minio_bucket - instance = started_cluster.instances["dummy"] + instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" - instance.query( - f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1" + exec_query_with_retry( + instance, + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", ) - table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_parquet', 'Parquet', 'a Int32, b String')" result = instance.query(f"SELECT count() FROM {table_function}") - assert int(result) == 5000000 + assert int(result) == 1000000 table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" exec_query_with_retry( instance, - f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1", + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1", ) - table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_orc', 'ORC', 'a Int32, b String')" - result = instance.query(f"SELECT count() FROM {table_function}") - assert int(result) == 5000000 - - instance.query("SYSTEM FLUSH LOGS") + table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_parquet', 'Parquet', 'a Int32, b String')" result = instance.query( - f"SELECT formatReadableSize(memory_usage) FROM system.query_log WHERE startsWith(query, 'SELECT count() FROM url') AND memory_usage > 0 ORDER BY event_time desc" + f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='50M'" ) - - result = result[: result.index(".")] - assert int(result) < 200 + assert int(result) == 1000000 def test_empty_file(started_cluster): diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.reference b/tests/queries/0_stateless/01710_minmax_count_projection.reference index 259d320a38a..bbcec98fb74 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.reference +++ b/tests/queries/0_stateless/01710_minmax_count_projection.reference @@ -2,7 +2,9 @@ 0 9998 5000 1 9999 5000 0 9998 5000 +0 9998 5000 1 +0 9998 5000 0 1 0 9999 @@ -14,6 +16,7 @@ 1 1 1 +2021-10-25 10:00:00 3 2021-10-27 10:00:00 3 \N 2021-10-27 10:00:00 3 0 2021-10-24 10:00:00 diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index a6c04725583..7ceff6a2662 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -10,11 +10,15 @@ set max_rows_to_read = 2, allow_experimental_projection_optimization = 1; select min(i), max(i), count() from d; select min(i), max(i), count() from d group by _partition_id order by _partition_id; select min(i), max(i), count() from d where _partition_value.1 = 0 group by _partition_id order by _partition_id; +select min(i), max(i), count() from d where moduloLegacy(i, 2) = 0 group by _partition_id order by _partition_id; select min(i), max(i), count() from d where _partition_value.1 = 10 group by _partition_id order by _partition_id; -- fuzz crash select min(i) from d where 1 = _partition_value.1; +-- fuzz crash https://github.com/ClickHouse/ClickHouse/issues/37151 +SELECT min(i), max(i), count() FROM d WHERE (_partition_value.1) = 0 GROUP BY ignore(bitTest(ignore(NULL), 65535), NULL, (_partition_value.1) = 7, '10.25', bitTest(NULL, -9223372036854775808), NULL, ignore(ignore(-2147483647, NULL)), 1024), _partition_id ORDER BY _partition_id ASC NULLS FIRST; + drop table d; drop table if exists has_final_mark; @@ -54,6 +58,9 @@ select min(dt), max(dt), count() from d where toDate(dt) >= '2021-10-25'; select min(dt), max(dt), count(toDate(dt) >= '2021-10-25') from d where toDate(dt) >= '2021-10-25'; select count() from d group by toDate(dt); +-- fuzz crash +SELECT min(dt), count(ignore(ignore(ignore(tupleElement(_partition_value, NULL) = NULL), NULL, NULL, NULL), 0, '10485.76', NULL)), max(dt), count(toDate(dt) >= '2021-10-25') FROM d WHERE toDate(dt) >= '2021-10-25'; + -- fuzz crash SELECT pointInEllipses(min(j), NULL), max(dt), count('0.0000000007') FROM d WHERE toDate(dt) >= '2021-10-25'; SELECT min(j) FROM d PREWHERE ceil(j) <= 0; diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.reference b/tests/queries/0_stateless/02241_join_rocksdb_bs.reference new file mode 100644 index 00000000000..8416a2991c1 --- /dev/null +++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.reference @@ -0,0 +1,68 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 new file mode 100644 index 00000000000..6121db6d6a2 --- /dev/null +++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 @@ -0,0 +1,44 @@ +-- Tags: use-rocksdb, long + +SET join_algorithm = 'direct'; + +{% for table_size in [10, 65555, 100000] -%} +DROP TABLE IF EXISTS rdb_{{ table_size }}; +{% endfor -%} + +{% for table_size in [10, 65555, 100000] -%} + +CREATE TABLE rdb_{{ table_size }} (key UInt64, value String) ENGINE = EmbeddedRocksDB PRIMARY KEY (key); +INSERT INTO rdb_{{ table_size }} + SELECT (sipHash64(number) % {{ table_size }}) as key, ('val' || toString(key)) AS value + FROM numbers_mt({{ table_size }}); + +{% for block_size in [10, 11, 128, 129, 65505, 65506, 70000] -%} + +{% if block_size * 5000 > table_size -%} + +SET max_block_size = {{ block_size }}; + +{% for right_size in [table_size // 2, table_size + table_size // 4 + 1] -%} + +SELECT count() == (SELECT count() FROM rdb_{{ table_size }} WHERE key < {{ right_size }}) +FROM (SELECT number as k FROM numbers_mt({{ right_size }})) as t1 +INNER JOIN rdb_{{ table_size }} as rdb +ON rdb.key == t1.k; + +SELECT count() == {{ right_size }} and countIf(value != '') == (SELECT count() FROM rdb_{{ table_size }} WHERE key < {{ right_size }}) +FROM (SELECT number as k FROM numbers_mt({{ right_size }})) as t1 +LEFT JOIN rdb_{{ table_size }} as rdb +ON rdb.key == t1.k; + +{% endfor -%} + +{% endif -%} + +{% endfor -%} +{% endfor -%} + +{% for table_size in [10, 65555, 100000] -%} +DROP TABLE IF EXISTS rdb_{{ table_size }}; +{% endfor -%} + diff --git a/tests/queries/0_stateless/02242_join_rocksdb.reference b/tests/queries/0_stateless/02242_join_rocksdb.reference new file mode 100644 index 00000000000..b1f7307ff4f --- /dev/null +++ b/tests/queries/0_stateless/02242_join_rocksdb.reference @@ -0,0 +1,60 @@ +-- key rename +0 0 [0,1] val20 +2 2 [2,3] val22 +3 3 [3,4] val23 +6 6 [6,7] val26 +7 7 [7,8] val27 +8 8 [8,9] val28 +9 9 [9,10] val29 +-- using +0 [0,1] val20 +2 [2,3] val22 +3 [3,4] val23 +6 [6,7] val26 +7 [7,8] val27 +8 [8,9] val28 +9 [9,10] val29 +-- join_use_nulls left +0 0 Nullable(String) val20 +1 \N Nullable(String) \N +2 2 Nullable(String) val22 +3 3 Nullable(String) val23 +4 \N Nullable(String) \N +5 \N Nullable(String) \N +6 6 Nullable(String) val26 +7 7 Nullable(String) val27 +8 8 Nullable(String) val28 +9 9 Nullable(String) val29 +-- join_use_nulls inner +0 0 String val20 +2 2 String val22 +3 3 String val23 +6 6 String val26 +7 7 String val27 +8 8 String val28 +9 9 String val29 +-- columns subset +val20 + +val22 +val23 + + +val26 +val27 +val28 +val29 +--- key types +0 0 [0,1] val20 +2 2 [2,3] val22 +3 3 [3,4] val23 +6 6 [6,7] val26 +7 7 [7,8] val27 +8 8 [8,9] val28 +9 9 [9,10] val29 +--- totals +0 16 val28 +1 19 val29 + +0 35 val29 +--- diff --git a/tests/queries/0_stateless/02242_join_rocksdb.sql b/tests/queries/0_stateless/02242_join_rocksdb.sql new file mode 100644 index 00000000000..1759311163b --- /dev/null +++ b/tests/queries/0_stateless/02242_join_rocksdb.sql @@ -0,0 +1,64 @@ +-- Tags: use-rocksdb + +DROP TABLE IF EXISTS rdb; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE rdb (key UInt32, value Array(UInt32), value2 String) ENGINE = EmbeddedRocksDB PRIMARY KEY (key); +INSERT INTO rdb + SELECT + toUInt32(sipHash64(number) % 10) as key, + [key, key+1] as value, + ('val2' || toString(key)) as value2 + FROM numbers_mt(10); + +CREATE TABLE t1 (k UInt32) ENGINE = TinyLog; +INSERT INTO t1 SELECT number as k from numbers_mt(10); + +CREATE TABLE t2 (k UInt16) ENGINE = TinyLog; +INSERT INTO t2 SELECT number as k from numbers_mt(10); + +SET join_algorithm = 'direct'; + +SELECT '-- key rename'; +SELECT * FROM (SELECT k as key FROM t2) as t2 INNER JOIN rdb ON rdb.key == t2.key ORDER BY key; + +SELECT '-- using'; +SELECT * FROM (SELECT k as key FROM t2) as t2 INNER JOIN rdb USING key ORDER BY key; + +SELECT '-- join_use_nulls left'; +SELECT k, key, toTypeName(value2), value2 FROM t2 LEFT JOIN rdb ON rdb.key == t2.k ORDER BY k SETTINGS join_use_nulls = 1; + +SELECT '-- join_use_nulls inner'; +SELECT k, key, toTypeName(value2), value2 FROM t2 INNER JOIN rdb ON rdb.key == t2.k ORDER BY k SETTINGS join_use_nulls = 1; + +SELECT '-- columns subset'; +SELECT value2 FROM t2 LEFT JOIN rdb ON rdb.key == t2.k ORDER BY k; + +SELECT '--- key types'; +SELECT * FROM t2 INNER JOIN rdb ON rdb.key == t2.k ORDER BY rdb.key; + +-- can't promote right table type +SELECT * FROM (SELECT toUInt64(k) as k FROM t2) as t2 INNER JOIN rdb ON rdb.key == t2.k; -- { serverError TYPE_MISMATCH } +-- TODO: support fallcack when right table key type can't be changed +-- SELECT * FROM (SELECT toUInt64(k) as k FROM t2) as t2 INNER JOIN rdb ON rdb.key == t2.k FORMAT Null SETTINGS join_algorithm = 'direct,hash'; + +SELECT '--- totals'; +SELECT rdb.key % 2, sum(k), max(value2) FROM t2 INNER JOIN rdb ON rdb.key == t2.k GROUP BY (rdb.key % 2) WITH TOTALS; + +SELECT '---'; +SELECT * FROM t1 RIGHT JOIN rdb ON rdb.key == t1.k; -- { serverError UNSUPPORTED_METHOD } +SELECT * FROM t1 RIGHT JOIN rdb ON rdb.key == t1.k FORMAT Null SETTINGS join_algorithm = 'direct,hash'; + +SELECT * FROM t1 FULL JOIN rdb ON rdb.key == t1.k; -- { serverError UNSUPPORTED_METHOD } +SELECT * FROM t1 FULL JOIN rdb ON rdb.key == t1.k FORMAT Null SETTINGS join_algorithm = 'direct,hash'; + +SELECT * FROM t1 INNER JOIN rdb ON rdb.key + 1 == t1.k; -- { serverError UNSUPPORTED_METHOD } +SELECT * FROM t1 INNER JOIN rdb ON rdb.key + 1 == t1.k FORMAT Null SETTINGS join_algorithm = 'direct,hash'; + +SELECT * FROM t1 INNER JOIN (SELECT * FROM rdb) AS rdb ON rdb.key == t1.k; -- { serverError UNSUPPORTED_METHOD } +SELECT * FROM t1 INNER JOIN (SELECT * FROM rdb) AS rdb ON rdb.key == t1.k FORMAT Null SETTINGS join_algorithm = 'direct,hash'; + +DROP TABLE IF EXISTS rdb; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/02245_s3_schema_desc.sql b/tests/queries/0_stateless/02245_s3_schema_desc.sql index 2cd362ff233..8c12d196800 100644 --- a/tests/queries/0_stateless/02245_s3_schema_desc.sql +++ b/tests/queries/0_stateless/02245_s3_schema_desc.sql @@ -11,4 +11,4 @@ desc s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test desc s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto'); -SELECT * FROM s3(decodeURLComponent(NULL), [NULL]); --{serverError 170} +SELECT * FROM s3(decodeURLComponent(NULL), [NULL]); --{serverError BAD_ARGUMENTS} diff --git a/tests/queries/0_stateless/02322_sql_insert_format.reference b/tests/queries/0_stateless/02322_sql_insert_format.reference new file mode 100644 index 00000000000..e64ef587fa7 --- /dev/null +++ b/tests/queries/0_stateless/02322_sql_insert_format.reference @@ -0,0 +1,23 @@ +INSERT INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (1, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (2, 2, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (3, 0, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (4, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (2, 2, 'Hello'), (3, 0, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (4, 1, 'Hello'); +INSERT INTO table VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +REPLACE INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +INSERT INTO test (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +INSERT INTO test (x, y, z) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +0 0 Hello +1 1 Hello +2 2 Hello +3 0 Hello +4 1 Hello +0 0 Hello +1 1 Hello +2 2 Hello +3 0 Hello +4 1 Hello diff --git a/tests/queries/0_stateless/02322_sql_insert_format.sql b/tests/queries/0_stateless/02322_sql_insert_format.sql new file mode 100644 index 00000000000..34cde1e56b6 --- /dev/null +++ b/tests/queries/0_stateless/02322_sql_insert_format.sql @@ -0,0 +1,13 @@ +-- Tags: no-parallel + +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_max_batch_size=1; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_max_batch_size=2; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_include_column_names=0; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_use_replace=1; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test'; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test', output_format_sql_insert_quote_names=0; +insert into function file(02322_data.sql, 'SQLInsert') select number as x, number % 3 as y, 'Hello' as z from numbers(5) settings output_format_sql_insert_max_batch_size=2, output_format_sql_insert_quote_names=0, engine_file_truncate_on_insert=1; +select * from file(02322_data.sql, 'MySQLDump'); +insert into function file(02322_data.sql, 'SQLInsert') select number, number % 3, 'Hello' from numbers(5) settings output_format_sql_insert_max_batch_size=2, engine_file_truncate_on_insert=1; +select * from file(02322_data.sql, 'MySQLDump'); diff --git a/tests/queries/0_stateless/02324_map_combinator_bug.reference b/tests/queries/0_stateless/02324_map_combinator_bug.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/02324_map_combinator_bug.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/02324_map_combinator_bug.sql b/tests/queries/0_stateless/02324_map_combinator_bug.sql new file mode 100644 index 00000000000..b4e039b95ab --- /dev/null +++ b/tests/queries/0_stateless/02324_map_combinator_bug.sql @@ -0,0 +1,28 @@ +-- Tags: no-backward-compatibility-check:22.6 + +DROP TABLE IF EXISTS segfault; +DROP TABLE IF EXISTS segfault_mv; + +CREATE TABLE segfault +( + id UInt32, + uuid UUID, + tags_ids Array(UInt32) +) ENGINE = MergeTree() +ORDER BY (id); + +CREATE MATERIALIZED VIEW segfault_mv + ENGINE = AggregatingMergeTree() + ORDER BY (id) +AS SELECT + id, + uniqState(uuid) as uniq_uuids, + uniqMapState(CAST((tags_ids, arrayMap(_ -> toString(uuid), tags_ids)), 'Map(UInt32, String)')) as uniq_tags_ids +FROM segfault +GROUP BY id; + +INSERT INTO segfault SELECT * FROM generateRandom('id UInt32, uuid UUID, c Array(UInt32)', 10, 5, 5) LIMIT 100; +INSERT INTO segfault SELECT * FROM generateRandom('id UInt32, uuid UUID, c Array(UInt32)', 10, 5, 5) LIMIT 100; +INSERT INTO segfault SELECT * FROM generateRandom('id UInt32, uuid UUID, c Array(UInt32)', 10, 5, 5) LIMIT 100; + +SELECT ignore(CAST((arrayMap(k -> toString(k), mapKeys(uniqMapMerge(uniq_tags_ids) AS m)), mapValues(m)), 'Map(String, UInt32)')) FROM segfault_mv; diff --git a/tests/queries/0_stateless/02352_interactive_queries_from_file.expect b/tests/queries/0_stateless/02352_interactive_queries_from_file.expect new file mode 100755 index 00000000000..d15b804b0b9 --- /dev/null +++ b/tests/queries/0_stateless/02352_interactive_queries_from_file.expect @@ -0,0 +1,45 @@ +#!/usr/bin/expect -f +# tags: long, no-parallel + +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + +log_user 0 +set timeout 20 +match_max 100000 + +expect_after { + # Do not ignore eof from expect + eof { exp_continue } + # A default timeout action is to do nothing, change it to fail + timeout { exit 1 } +} + +spawn bash -c "echo 'select 1;\nselect 2;\nselect 3' > queries_02352" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --disable_suggestion" +expect ":) " + +send -- "\\i queries_02352\r" +expect "1" +expect "2" +expect "3" +expect ":) " +send -- "\\i queries_02352;\r" +expect "1" +expect "2" +expect "3" +expect ":) " +send -- " \\i queries_02352 ; \r" +expect "1" +expect "2" +expect "3" +expect ":) " +send -- " \\i queries_02352 ; \r" +expect "1" +expect "2" +expect "3" +expect ":) " + +send -- "exit\r" +expect eof diff --git a/tests/queries/0_stateless/02352_interactive_queries_from_file.reference b/tests/queries/0_stateless/02352_interactive_queries_from_file.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 2bfa98b80c7..cc22b712c62 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -20,6 +20,7 @@ CamelCase CapnProto CentOS ClickHouse +ClickHouse's Config Contrib Ctrl @@ -121,6 +122,7 @@ SATA SERIALIZABLE SIMD SMALLINT +SQLInsert SQLSTATE SSSE Schemas @@ -411,6 +413,7 @@ simdjson skippingerrors sparsehash sql +sqlinsert src stacktraces statbox diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index d3a8323b81f..5d1d0f8a491 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -62,7 +62,7 @@ void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & pa promise->set_value(); }; - zookeeper.list(path, list_callback, nullptr); + zookeeper.list(path, ListRequestType::ALL, list_callback, nullptr); future.get(); while (!children.empty())