diff --git a/CMakeLists.txt b/CMakeLists.txt index e765ca02a4f..e0fa6d8e197 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,16 +80,16 @@ include (cmake/find/ccache.cmake) # ccache ignore it. option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF) if (ENABLE_CHECK_HEAVY_BUILDS) - # set DATA (since RSS does not work since 2.6.x+) to 2G + # set DATA (since RSS does not work since 2.6.x+) to 5G set (RLIMIT_DATA 5000000000) # set VIRT (RLIMIT_AS) to 10G (DATA*10) set (RLIMIT_AS 10000000000) - # set CPU time limit to 600 seconds - set (RLIMIT_CPU 600) + # set CPU time limit to 1000 seconds + set (RLIMIT_CPU 1000) # gcc10/gcc10/clang -fsanitize=memory is too heavy if (SANITIZE STREQUAL "memory" OR COMPILER_GCC) - set (RLIMIT_DATA 10000000000) + set (RLIMIT_DATA 10000000000) # 10G endif() set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) diff --git a/base/mysqlxx/Pool.h b/base/mysqlxx/Pool.h index 08d8b85b4ac..f542c3d3b76 100644 --- a/base/mysqlxx/Pool.h +++ b/base/mysqlxx/Pool.h @@ -189,7 +189,7 @@ public: ~Pool(); /// Allocates connection. - Entry get(uint64_t wait_timeout); + Entry get(uint64_t wait_timeout = UINT64_MAX); /// Allocates connection. /// If database is not accessible, returns empty Entry object. diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index 816164bef10..84ac29991ab 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -1,8 +1,10 @@ -if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") - set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "") +if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") + set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF) +else() + set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON) endif() -option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ON) +option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT}) if (NOT ENABLE_EMBEDDED_COMPILER) set (USE_EMBEDDED_COMPILER 0) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e6e098a05b3..383a8510035 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -206,12 +206,14 @@ elseif(GTEST_SRC_DIR) target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0) endif() -if (USE_EMBEDDED_COMPILER) +function(add_llvm) # ld: unknown option: --color-diagnostics if (APPLE) set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") endif () + # Do not adjust RPATH in llvm, since then it will not be able to find libcxx/libcxxabi/libunwind + set (CMAKE_INSTALL_RPATH "ON") set (LLVM_ENABLE_EH 1 CACHE INTERNAL "") set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "") set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "") @@ -219,13 +221,12 @@ if (USE_EMBEDDED_COMPILER) # Need to use C++17 since the compilation is not possible with C++20 currently, due to ambiguous operator != etc. # LLVM project will set its default value for the -std=... but our global setting from CMake will override it. - set (CMAKE_CXX_STANDARD_bak ${CMAKE_CXX_STANDARD}) set (CMAKE_CXX_STANDARD 17) add_subdirectory (llvm/llvm) - - set (CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD_bak}) - unset (CMAKE_CXX_STANDARD_bak) +endfunction() +if (USE_EMBEDDED_COMPILER) + add_llvm() endif () if (USE_INTERNAL_LIBGSASL_LIBRARY) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 4e70d382b36..1dd87fe80ae 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -3,10 +3,17 @@ # Provides: clickhouse-server # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Required-Start: $network -# Required-Stop: $network +# Should-Start: $time $network +# Should-Stop: $network # Short-Description: Yandex clickhouse-server daemon ### END INIT INFO +# +# NOTES: +# - Should-* -- script can start if the listed facilities are missing, unlike Required-* +# +# For the documentation [1]: +# +# [1]: https://wiki.debian.org/LSBInitScripts CLICKHOUSE_USER=clickhouse CLICKHOUSE_GROUP=${CLICKHOUSE_USER} diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index b9681f9279e..bc19235cb3a 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -1,7 +1,12 @@ [Unit] Description=ClickHouse Server (analytic DBMS for big data) Requires=network-online.target -After=network-online.target +# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure +# that the time was adjusted already, if you use systemd-timesyncd you are +# safe, but if you use ntp or some other daemon, you should configure it +# additionaly. +After=time-sync.target network-online.target +Wants=time-sync.target [Service] Type=simple @@ -16,4 +21,5 @@ LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE [Install] +# ClickHouse should not start from the rescue shell (rescue.target). WantedBy=multi-user.target diff --git a/docker/images.json b/docker/images.json index e2e22468596..96424bb9432 100644 --- a/docker/images.json +++ b/docker/images.json @@ -1,12 +1,12 @@ { "docker/packager/deb": { - "name": "yandex/clickhouse-deb-builder", + "name": "clickhouse/deb-builder", "dependent": [ "docker/packager/unbundled" ] }, "docker/packager/binary": { - "name": "yandex/clickhouse-binary-builder", + "name": "clickhouse/binary-builder", "dependent": [ "docker/test/split_build_smoke_test", "docker/test/pvs", @@ -14,155 +14,150 @@ ] }, "docker/packager/unbundled": { - "name": "yandex/clickhouse-unbundled-builder", + "name": "clickhouse/unbundled-builder", "dependent": [] }, "docker/test/compatibility/centos": { - "name": "yandex/clickhouse-test-old-centos", + "name": "clickhouse/test-old-centos", "dependent": [] }, "docker/test/compatibility/ubuntu": { - "name": "yandex/clickhouse-test-old-ubuntu", + "name": "clickhouse/test-old-ubuntu", "dependent": [] }, "docker/test/integration/base": { - "name": "yandex/clickhouse-integration-test", + "name": "clickhouse/integration-test", "dependent": [] }, "docker/test/fuzzer": { - "name": "yandex/clickhouse-fuzzer", + "name": "clickhouse/fuzzer", "dependent": [] }, "docker/test/performance-comparison": { - "name": "yandex/clickhouse-performance-comparison", + "name": "clickhouse/performance-comparison", "dependent": [] }, "docker/test/pvs": { - "name": "yandex/clickhouse-pvs-test", + "name": "clickhouse/pvs-test", "dependent": [] }, "docker/test/stateless": { - "name": "yandex/clickhouse-stateless-test", + "name": "clickhouse/stateless-test", "dependent": [ "docker/test/stateful", "docker/test/coverage", "docker/test/unit" ] }, - "docker/test/stateless_pytest": { - "name": "yandex/clickhouse-stateless-pytest", - "dependent": [] - }, "docker/test/stateful": { - "name": "yandex/clickhouse-stateful-test", + "name": "clickhouse/stateful-test", "dependent": [ "docker/test/stress" ] }, "docker/test/coverage": { - "name": "yandex/clickhouse-test-coverage", + "name": "clickhouse/test-coverage", "dependent": [] }, "docker/test/unit": { - "name": "yandex/clickhouse-unit-test", + "name": "clickhouse/unit-test", "dependent": [] }, "docker/test/stress": { - "name": "yandex/clickhouse-stress-test", + "name": "clickhouse/stress-test", "dependent": [] }, "docker/test/split_build_smoke_test": { - "name": "yandex/clickhouse-split-build-smoke-test", + "name": "clickhouse/split-build-smoke-test", "dependent": [] }, "docker/test/codebrowser": { - "name": "yandex/clickhouse-codebrowser", + "name": "clickhouse/codebrowser", "dependent": [] }, "docker/test/integration/runner": { - "name": "yandex/clickhouse-integration-tests-runner", + "name": "clickhouse/integration-tests-runner", "dependent": [] }, "docker/test/testflows/runner": { - "name": "yandex/clickhouse-testflows-runner", + "name": "clickhouse/testflows-runner", "dependent": [] }, "docker/test/fasttest": { - "name": "yandex/clickhouse-fasttest", + "name": "clickhouse/fasttest", "dependent": [] }, "docker/test/style": { - "name": "yandex/clickhouse-style-test", + "name": "clickhouse/style-test", "dependent": [] }, "docker/test/integration/s3_proxy": { - "name": "yandex/clickhouse-s3-proxy", + "name": "clickhouse/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "name": "yandex/clickhouse-python-bottle", + "name": "clickhouse/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "name": "yandex/clickhouse-integration-helper", + "name": "clickhouse/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "name": "yandex/clickhouse-mysql-golang-client", + "name": "clickhouse/mysql-golang-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "name": "yandex/clickhouse-mysql-java-client", + "name": "clickhouse/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "name": "yandex/clickhouse-mysql-js-client", + "name": "clickhouse/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "name": "yandex/clickhouse-mysql-php-client", + "name": "clickhouse/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "name": "yandex/clickhouse-postgresql-java-client", + "name": "clickhouse/postgresql-java-client", "dependent": [] }, "docker/test/integration/kerberos_kdc": { - "name": "yandex/clickhouse-kerberos-kdc", + "name": "clickhouse/kerberos-kdc", "dependent": [] }, "docker/test/base": { - "name": "yandex/clickhouse-test-base", + "name": "clickhouse/test-base", "dependent": [ "docker/test/stateless", "docker/test/stateless_unbundled", - "docker/test/stateless_pytest", "docker/test/integration/base", "docker/test/fuzzer", "docker/test/keeper-jepsen" ] }, "docker/packager/unbundled": { - "name": "yandex/clickhouse-unbundled-builder", + "name": "clickhouse/unbundled-builder", "dependent": [ "docker/test/stateless_unbundled" ] }, "docker/test/stateless_unbundled": { - "name": "yandex/clickhouse-stateless-unbundled-test", + "name": "clickhouse/stateless-unbundled-test", "dependent": [ ] }, "docker/test/integration/kerberized_hadoop": { - "name": "yandex/clickhouse-kerberized-hadoop", + "name": "clickhouse/kerberized-hadoop", "dependent": [] }, "docker/test/sqlancer": { - "name": "yandex/clickhouse-sqlancer-test", + "name": "clickhouse/sqlancer-test", "dependent": [] }, "docker/test/keeper-jepsen": { - "name": "yandex/clickhouse-keeper-jepsen-test", + "name": "clickhouse/keeper-jepsen-test", "dependent": [] } } diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index f5d496ce97f..832a913e34c 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-binary-builder . +# docker build -t clickhouse/binary-builder . FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 22bba94f250..0a535ed2a71 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-deb-builder . +# docker build -t clickhouse/deb-builder . FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 diff --git a/docker/packager/packager b/docker/packager/packager index f37d64e9949..ec423bf2c76 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -9,9 +9,9 @@ import sys SCRIPT_PATH = os.path.realpath(__file__) IMAGE_MAP = { - "deb": "yandex/clickhouse-deb-builder", - "binary": "yandex/clickhouse-binary-builder", - "unbundled": "yandex/clickhouse-unbundled-builder" + "deb": "clickhouse/deb-builder", + "binary": "clickhouse/binary-builder", + "unbundled": "clickhouse/unbundled-builder" } def check_image_exists_locally(image_name): diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index 1d0eee155ef..4fe093a0751 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-unbundled-builder . -FROM yandex/clickhouse-deb-builder +# docker build -t clickhouse/unbundled-builder . +FROM clickhouse/deb-builder RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ && wget -nv -O /tmp/arrow-keyring.deb "https://apache.jfrog.io/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-${CODENAME}.deb" \ diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 88c9e1ae06e..a0345406400 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-test-base . +# docker build -t clickhouse/test-base . FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index ae2aafa76c8..b87ad9336c9 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -1,6 +1,6 @@ -# docker build --network=host -t yandex/clickhouse-codebrowser . -# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output yandex/clickhouse-codebrowser -FROM yandex/clickhouse-binary-builder +# docker build --network=host -t clickhouse/codebrowser . +# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser +FROM clickhouse/binary-builder RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/compatibility/centos/Dockerfile b/docker/test/compatibility/centos/Dockerfile index 0ef119d1bb1..628609e374f 100644 --- a/docker/test/compatibility/centos/Dockerfile +++ b/docker/test/compatibility/centos/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-test-old-centos . +# docker build -t clickhouse/test-old-centos . FROM centos:5 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/compatibility/ubuntu/Dockerfile b/docker/test/compatibility/ubuntu/Dockerfile index 28f89e47b95..ddd0a76bd44 100644 --- a/docker/test/compatibility/ubuntu/Dockerfile +++ b/docker/test/compatibility/ubuntu/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-test-old-ubuntu . +# docker build -t clickhouse/test-old-ubuntu . FROM ubuntu:12.04 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/coverage/Dockerfile b/docker/test/coverage/Dockerfile index 681f65e0f6f..ccf0bbc7c83 100644 --- a/docker/test/coverage/Dockerfile +++ b/docker/test/coverage/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-test-coverage . -FROM yandex/clickhouse-stateless-test +# docker build -t clickhouse/test-coverage . +FROM clickhouse/stateless-test RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 9443cbf496e..1daab8b80f1 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-fasttest . +# docker build -t clickhouse/fasttest . FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index 9a96ac1dfa7..6444e745c47 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-fuzzer . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/fuzzer . +FROM clickhouse/test-base ENV LANG=C.UTF-8 ENV TZ=Europe/Moscow @@ -36,5 +36,5 @@ CMD set -o pipefail \ && cd /workspace \ && /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log -# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-fuzzer +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/fuzzer diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 344c1b9a698..519c64297e5 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-integration-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/integration-test . +FROM clickhouse/test-base SHELL ["/bin/bash", "-c"] diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile index 922eb2c6f22..6a093081bf2 100644 --- a/docker/test/integration/helper_container/Dockerfile +++ b/docker/test/integration/helper_container/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-integration-helper . +# docker build -t clickhouse/integration-helper . # Helper docker container to run iptables without sudo FROM alpine diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 6a2fd96e7a7..11da590f901 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-kerberized-hadoop . +# docker build -t clickhouse/kerberized-hadoop . FROM sequenceiq/hadoop-docker:2.7.0 RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo diff --git a/docker/test/integration/kerberos_kdc/Dockerfile b/docker/test/integration/kerberos_kdc/Dockerfile index ea231b1191d..a203c33a331 100644 --- a/docker/test/integration/kerberos_kdc/Dockerfile +++ b/docker/test/integration/kerberos_kdc/Dockerfile @@ -1,9 +1,9 @@ -# docker build -t yandex/clickhouse-kerberos-kdc . +# docker build -t clickhouse/kerberos-kdc . +FROM centos:6 -FROM centos:6.6 -# old OS to make is faster and smaller +RUN sed -i '/^mirrorlist/s/^/#/;/^#baseurl/{s/#//;s/mirror.centos.org\/centos\/$releasever/vault.centos.org\/6.10/}' /etc/yum.repos.d/*B* -RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation +RUN yum install -y ca-certificates krb5-server krb5-libs krb5-auth-dialog krb5-workstation EXPOSE 88 749 diff --git a/docker/test/integration/mysql_golang_client/Dockerfile b/docker/test/integration/mysql_golang_client/Dockerfile index 4380383d1fb..68b0aaab42c 100644 --- a/docker/test/integration/mysql_golang_client/Dockerfile +++ b/docker/test/integration/mysql_golang_client/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t yandex/clickhouse-mysql-golang-client . +# docker build -t clickhouse/mysql-golang-client . # MySQL golang client docker container -FROM golang:1.12.2 +FROM golang:1.13 RUN go get "github.com/go-sql-driver/mysql" diff --git a/docker/test/integration/mysql_java_client/Dockerfile b/docker/test/integration/mysql_java_client/Dockerfile index fcb6a39f33b..0abf50cd493 100644 --- a/docker/test/integration/mysql_java_client/Dockerfile +++ b/docker/test/integration/mysql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-mysql-java-client . +# docker build -t clickhouse/mysql-java-client . # MySQL Java client docker container FROM ubuntu:18.04 diff --git a/docker/test/integration/mysql_js_client/Dockerfile b/docker/test/integration/mysql_js_client/Dockerfile index 4f12de004ac..b1397b40d38 100644 --- a/docker/test/integration/mysql_js_client/Dockerfile +++ b/docker/test/integration/mysql_js_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-mysql-js-client . +# docker build -t clickhouse/mysql-js-client . # MySQL JavaScript client docker container FROM node:8 diff --git a/docker/test/integration/mysql_php_client/Dockerfile b/docker/test/integration/mysql_php_client/Dockerfile index e2ceb62f44f..0fb77bf8ffb 100644 --- a/docker/test/integration/mysql_php_client/Dockerfile +++ b/docker/test/integration/mysql_php_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-mysql-php-client . +# docker build -t clickhouse/mysql-php-client . # MySQL PHP client docker container FROM php:7.3-cli diff --git a/docker/test/integration/postgresql_java_client/Dockerfile b/docker/test/integration/postgresql_java_client/Dockerfile index eab236c9590..f5484028ec9 100644 --- a/docker/test/integration/postgresql_java_client/Dockerfile +++ b/docker/test/integration/postgresql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-postgresql-java-client . +# docker build -t clickhouse/postgresql-java-client . # PostgreSQL Java client docker container FROM ubuntu:18.04 diff --git a/docker/test/integration/resolver/Dockerfile b/docker/test/integration/resolver/Dockerfile index b0efb4b46d5..01b9b777614 100644 --- a/docker/test/integration/resolver/Dockerfile +++ b/docker/test/integration/resolver/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-python-bottle . +# docker build -t clickhouse/python-bottle . # Helper docker container to run python bottle apps FROM python:3 diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index ef333e461c5..06e1f64ced2 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-integration-tests-runner . +# docker build -t clickhouse/integration-tests-runner . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml b/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml index a65ef629df6..b3686adc21c 100644 --- a/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml +++ b/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml @@ -1,7 +1,7 @@ version: '2.3' services: bridge1: - image: yandex/clickhouse-jdbc-bridge + image: clickhouse/jdbc-bridge command: | /bin/bash -c 'cat << EOF > config/datasources/self.json { diff --git a/docker/test/integration/runner/compose/docker_compose_keeper.yml b/docker/test/integration/runner/compose/docker_compose_keeper.yml index e11a13e6eab..134ffbff1f7 100644 --- a/docker/test/integration/runner/compose/docker_compose_keeper.yml +++ b/docker/test/integration/runner/compose/docker_compose_keeper.yml @@ -1,7 +1,7 @@ version: '2.3' services: zoo1: - image: ${image:-yandex/clickhouse-integration-test} + image: ${image:-clickhouse/integration-test} restart: always user: ${user:-} volumes: @@ -31,7 +31,7 @@ services: - inet6 - rotate zoo2: - image: ${image:-yandex/clickhouse-integration-test} + image: ${image:-clickhouse/integration-test} restart: always user: ${user:-} volumes: @@ -61,7 +61,7 @@ services: - inet6 - rotate zoo3: - image: ${image:-yandex/clickhouse-integration-test} + image: ${image:-clickhouse/integration-test} restart: always user: ${user:-} volumes: diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml index b09e75a8515..88be3e45085 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml @@ -4,7 +4,7 @@ services: kerberizedhdfs1: cap_add: - DAC_READ_SEARCH - image: yandex/clickhouse-kerberized-hadoop:16621 + image: clickhouse/kerberized-hadoop hostname: kerberizedhdfs1 restart: always volumes: @@ -22,7 +22,7 @@ services: entrypoint: /etc/bootstrap.sh -d hdfskerberos: - image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: hdfskerberos volumes: - ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index 081b90c4f27..d57e4e4d5be 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -50,7 +50,7 @@ services: - label:disable kafka_kerberos: - image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml index 96a5f8bdc31..4384c0868a5 100644 --- a/docker/test/integration/runner/compose/docker_compose_minio.yml +++ b/docker/test/integration/runner/compose/docker_compose_minio.yml @@ -19,14 +19,14 @@ services: # HTTP proxies for Minio. proxy1: - image: yandex/clickhouse-s3-proxy + image: clickhouse/s3-proxy expose: - "8080" # Redirect proxy port - "80" # Reverse proxy port - "443" # Reverse proxy port (secure) proxy2: - image: yandex/clickhouse-s3-proxy + image: clickhouse/s3-proxy expose: - "8080" - "80" @@ -34,7 +34,7 @@ services: # Empty container to run proxy resolver. resolver: - image: yandex/clickhouse-python-bottle + image: clickhouse/python-bottle expose: - "8080" tty: true diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml index a6a338eb6a8..56cc0410574 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: golang1: - image: yandex/clickhouse-mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} + image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml index 21d927df82c..eb5ffb01baa 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: java1: - image: yandex/clickhouse-mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} + image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml index dbd85cf2382..90939449c5f 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: mysqljs1: - image: yandex/clickhouse-mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} + image: clickhouse/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml index f24f5337a7e..408b8ff089a 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: php1: - image: yandex/clickhouse-mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} + image: clickhouse/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml b/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml index 38191f1bdd6..904bfffdfd5 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml @@ -1,6 +1,6 @@ version: '2.2' services: java: - image: yandex/clickhouse-postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} + image: clickhouse/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/s3_proxy/Dockerfile b/docker/test/integration/s3_proxy/Dockerfile index d8b1754fa71..5858218e4e4 100644 --- a/docker/test/integration/s3_proxy/Dockerfile +++ b/docker/test/integration/s3_proxy/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-s3-proxy . +# docker build -t clickhouse/s3-proxy . FROM nginx:alpine COPY run.sh /run.sh diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index 1a62d5e793f..5bb7f9433c2 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-keeper-jepsen-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/keeper-jepsen-test . +FROM clickhouse/test-base ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 1a61c4b274a..88b66d42ecb 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-performance-comparison . +# docker build -t clickhouse/performance-comparison . FROM ubuntu:18.04 ENV LANG=C.UTF-8 @@ -54,4 +54,4 @@ COPY * / # it gives '/bin/sh: 1: [bash,: not found' otherwise. CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\\(.*\\)nodes.*$/\\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] -# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison +# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/performance-comparison diff --git a/docker/test/performance-comparison/README.md b/docker/test/performance-comparison/README.md index 782644a81dd..75213fad077 100644 --- a/docker/test/performance-comparison/README.md +++ b/docker/test/performance-comparison/README.md @@ -116,7 +116,7 @@ pull requests (0 for master) manually. docker run --network=host --volume=$(pwd)/workspace:/workspace --volume=$(pwd)/output:/output [-e REF_PR={} -e REF_SHA={}] -e PR_TO_TEST={} -e SHA_TO_TEST={} - yandex/clickhouse-performance-comparison + clickhouse/performance-comparison ``` Then see the `report.html` in the `output` directory. diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile index 438f0bd07ec..17bb7d3662b 100644 --- a/docker/test/pvs/Dockerfile +++ b/docker/test/pvs/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t yandex/clickhouse-pvs-test . +# docker build -t clickhouse/pvs-test . -FROM yandex/clickhouse-binary-builder +FROM clickhouse/binary-builder RUN apt-get update --yes \ && apt-get install \ diff --git a/docker/test/split_build_smoke_test/Dockerfile b/docker/test/split_build_smoke_test/Dockerfile index 54a9eb17868..3cc2f26a507 100644 --- a/docker/test/split_build_smoke_test/Dockerfile +++ b/docker/test/split_build_smoke_test/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-split-build-smoke-test . -FROM yandex/clickhouse-binary-builder +# docker build -t clickhouse/split-build-smoke-test . +FROM clickhouse/binary-builder COPY run.sh /run.sh COPY process_split_build_smoke_test_result.py / diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 67236402352..e73fd03fb6d 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -1,9 +1,9 @@ -# docker build -t yandex/clickhouse-sqlancer-test . +# docker build -t clickhouse/sqlancer-test . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list -RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git openjdk-14-jdk maven python3 --yes --no-install-recommends +RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git default-jdk maven python3 --yes --no-install-recommends RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip RUN mkdir /sqlancer && \ cd /sqlancer && \ diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 07aad75a2ea..c237a712f52 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateful-test . -FROM yandex/clickhouse-stateless-test +# docker build -t clickhouse/stateful-test . +FROM clickhouse/stateless-test RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index b66fa055e7b..3b5edb2c869 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateless-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/stateless-test . +FROM clickhouse/test-base ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stateless_pytest/Dockerfile b/docker/test/stateless_pytest/Dockerfile index 947a70426d6..c1e47523f6d 100644 --- a/docker/test/stateless_pytest/Dockerfile +++ b/docker/test/stateless_pytest/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateless-pytest . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/stateless-pytest . +FROM clickhouse/test-base RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 53857a90ac7..dfe441e08a6 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateless-unbundled-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/stateless-unbundled-test . +FROM clickhouse/test-base ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index e1df32ec3d7..3fe1b790d5a 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stress-test . -FROM yandex/clickhouse-stateful-test +# docker build -t clickhouse/stress-test . +FROM clickhouse/stateful-test RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stress/README.md b/docker/test/stress/README.md index f747996fa2d..b1519e7968d 100644 --- a/docker/test/stress/README.md +++ b/docker/test/stress/README.md @@ -6,7 +6,7 @@ Usage: ``` $ ls $HOME/someclickhouse clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb -$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output yandex/clickhouse-stress-test +$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test Selecting previously unselected package clickhouse-common-static. (Reading database ... 14442 files and directories currently installed.) ... diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index c0b3b0102cf..33cdb9db57a 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-style-test . +# docker build -t clickhouse/style-test . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/test_runner.sh b/docker/test/test_runner.sh index cd6367b2964..0c99c8c2b32 100755 --- a/docker/test/test_runner.sh +++ b/docker/test/test_runner.sh @@ -49,7 +49,7 @@ fi # Build server image (optional) from local packages if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then - CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" + CLICKHOUSE_SERVER_IMAGE="clickhouse/server:local" if [ "${CLICKHOUSE_PACKAGES_ARG}" != "${NO_REBUILD_FLAG}" ]; then docker build --network=host \ diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 81d431635b7..91d0eb844d9 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-testflows-runner . +# docker build -t clickhouse/testflows-runner . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index e111611eecd..20d67773363 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-unit-test . -FROM yandex/clickhouse-stateless-test +# docker build -t clickhouse/unit-test . +FROM clickhouse/stateless-test RUN apt-get install gdb diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 7249e24aff9..e494e9aec6a 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -210,4 +210,4 @@ ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file- ## See also -- [S3 table function](../../../sql-reference/table-functions/s3.md) +- [s3 table function](../../../sql-reference/table-functions/s3.md) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 7bd08549cb8..6c5767d28e0 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3499,6 +3499,30 @@ Possible values: Default value: `0`. +## replication_alter_partitions_sync {#replication-alter-partitions-sync} + +Allows to set up waiting for actions to be executed on replicas by [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) or [TRUNCATE](../../sql-reference/statements/truncate.md) queries. + +Possible values: + +- 0 — Do not wait. +- 1 — Wait for own execution. +- 2 — Wait for everyone. + +Default value: `1`. + +## replication_wait_for_inactive_replica_timeout {#replication-wait-for-inactive-replica-timeout} + +Specifies how long (in seconds) to wait for inactive replicas to execute [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) or [TRUNCATE](../../sql-reference/statements/truncate.md) queries. + +Possible values: + +- 0 — Do not wait. +- Negative integer — Wait for unlimited time. +- Positive integer — The number of seconds to wait. + +Default value: `120` seconds. + ## regexp_max_matches_per_row {#regexp-max-matches-per-row} Sets the maximum number of matches for a single regular expression per row. Use it to protect against memory overload when using greedy regular expression in the [extractAllGroupsHorizontal](../../sql-reference/functions/string-search-functions.md#extractallgroups-horizontal) function. diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index c22f041e0c3..69dd14da1bf 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -87,7 +87,23 @@ The function is using uppercase letters `A-F` and not using any prefixes (like ` For integer arguments, it prints hex digits (“nibbles”) from the most significant to least significant (big-endian or “human-readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if the leading digit is zero. -**Example** +Values of type [Date](../../sql-reference/data-types/date.md) and [DateTime](../../sql-reference/data-types/datetime.md) are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). + +For [String](../../sql-reference/data-types/string.md) and [FixedString](../../sql-reference/data-types/fixedstring.md), all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted. + +Values of [Float](../../sql-reference/data-types/float.md) and [Decimal](../../sql-reference/data-types/decimal.md) types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. + +**Arguments** + +- `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). + +**Returned value** + +- A string with the hexadecimal representation of the argument. + +Type: [String](../../sql-reference/data-types/string.md). + +**Examples** Query: @@ -101,28 +117,10 @@ Result: 01 ``` -Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). - -For `String` and `FixedString`, all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted. - -Values of floating point and Decimal types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. - -**Arguments** - -- `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). - -**Returned value** - -- A string with the hexadecimal representation of the argument. - -Type: `String`. - -**Example** - Query: ``` sql -SELECT hex(toFloat32(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat32(number)) AS hex_presentation FROM numbers(15, 2); ``` Result: @@ -137,7 +135,7 @@ Result: Query: ``` sql -SELECT hex(toFloat64(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat64(number)) AS hex_presentation FROM numbers(15, 2); ``` Result: @@ -210,52 +208,52 @@ Result: Returns a string containing the argument’s binary representation. -Alias: `BIN`. - **Syntax** ``` sql bin(arg) ``` +Alias: `BIN`. + For integer arguments, it prints bin digits from the most significant to least significant (big-endian or “human-readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints eight digits of every byte if the leading digit is zero. -**Example** +Values of type [Date](../../sql-reference/data-types/date.md) and [DateTime](../../sql-reference/data-types/datetime.md) are formatted as corresponding integers (the number of days since Epoch for `Date` and the value of Unix Timestamp for `DateTime`). -Query: +For [String](../../sql-reference/data-types/string.md) and [FixedString](../../sql-reference/data-types/fixedstring.md), all bytes are simply encoded as eight binary numbers. Zero bytes are not omitted. -``` sql -SELECT bin(1); -``` - -Result: - -``` text -00000001 -``` - -Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). - -For `String` and `FixedString`, all bytes are simply encoded as eight binary numbers. Zero bytes are not omitted. - -Values of floating-point and Decimal types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. +Values of [Float](../../sql-reference/data-types/float.md) and [Decimal](../../sql-reference/data-types/decimal.md) types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. **Arguments** -- `arg` — A value to convert to binary. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). +- `arg` — A value to convert to binary. [String](../../sql-reference/data-types/string.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md), or [DateTime](../../sql-reference/data-types/datetime.md). **Returned value** - A string with the binary representation of the argument. -Type: `String`. +Type: [String](../../sql-reference/data-types/string.md). -**Example** +**Examples** Query: ``` sql -SELECT bin(toFloat32(number)) as bin_presentation FROM numbers(15, 2); +SELECT bin(14); +``` + +Result: + +``` text +┌─bin(14)──┐ +│ 00001110 │ +└──────────┘ +``` + +Query: + +``` sql +SELECT bin(toFloat32(number)) AS bin_presentation FROM numbers(15, 2); ``` Result: @@ -270,7 +268,7 @@ Result: Query: ``` sql -SELECT bin(toFloat64(number)) as bin_presentation FROM numbers(15, 2); +SELECT bin(toFloat64(number)) AS bin_presentation FROM numbers(15, 2); ``` Result: @@ -284,14 +282,7 @@ Result: ## unbin {#unbinstr} -Performs the opposite operation of [bin](#bin). It interprets each pair of binary digits (in the argument) as a number and converts it to the byte represented by the number. The return value is a binary string (BLOB). - -If you want to convert the result to a number, you can use the [reverse](../../sql-reference/functions/string-functions.md#reverse) and [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#type-conversion-functions) functions. - -!!! note "Note" - If `unbin` is invoked from within the `clickhouse-client`, binary strings display using UTF-8. - -Alias: `UNBIN`. +Interprets each pair of binary digits (in the argument) as a number and converts it to the byte represented by the number. The functions performs the opposite operation to [bin](#bin). **Syntax** @@ -299,11 +290,18 @@ Alias: `UNBIN`. unbin(arg) ``` +Alias: `UNBIN`. + +For a numeric argument `unbin()` does not return the inverse of `bin()`. If you want to convert the result to a number, you can use the [reverse](../../sql-reference/functions/string-functions.md#reverse) and [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#reinterpretasuint8163264) functions. + +!!! note "Note" + If `unbin` is invoked from within the `clickhouse-client`, binary strings are displayed using UTF-8. + +Supports binary digits `0` and `1`. The number of binary digits does not have to be multiples of eight. If the argument string contains anything other than binary digits, some implementation-defined result is returned (an exception isn’t thrown). + **Arguments** -- `arg` — A string containing any number of binary digits. Type: [String](../../sql-reference/data-types/string.md). - -Supports binary digits `0-1`. The number of binary digits does not have to be multiples of eight. If the argument string contains anything other than binary digits, some implementation-defined result is returned (an exception isn’t thrown). For a numeric argument the inverse of bin(N) is not performed by unbin(). +- `arg` — A string containing any number of binary digits. [String](../../sql-reference/data-types/string.md). **Returned value** @@ -311,7 +309,7 @@ Supports binary digits `0-1`. The number of binary digits does not have to be mu Type: [String](../../sql-reference/data-types/string.md). -**Example** +**Examples** Query: @@ -330,14 +328,14 @@ Result: Query: ``` sql -SELECT reinterpretAsUInt64(reverse(unbin('1010'))) AS num; +SELECT reinterpretAsUInt64(reverse(unbin('1110'))) AS num; ``` Result: ``` text ┌─num─┐ -│ 10 │ +│ 14 │ └─────┘ ``` @@ -396,7 +394,7 @@ Result: Query: ``` sql -select bitPositionsToArray(toInt8(-1)) as bit_positions; +SELECT bitPositionsToArray(toInt8(-1)) AS bit_positions; ``` Result: diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index 54afd461e1d..47da4e6f3cc 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -59,6 +59,10 @@ A lambda function that accepts multiple arguments can also be passed to a higher For some functions the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed. +## User Defined Functions {#user-defined-functions} + +Custom functions can be created using the [CREATE FUNCTION](../statements/create/function.md) statement. To delete these functions use the [DROP FUNCTION](../statements/drop.md#drop-function) statement. + ## Error Handling {#error-handling} Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query. diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index ef5f5814017..6e91419b04f 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -78,7 +78,7 @@ mapAdd(arg1, arg2 [, ...]) **Arguments** -Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array), where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promote to the one type ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) or [Float64](../../sql-reference/data-types/float.md#float32-float64)). The common promoted type is used as a type for the result array. +Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array), where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promoted to the one type ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) or [Float64](../../sql-reference/data-types/float.md#float32-float64)). The common promoted type is used as a type for the result array. **Returned value** @@ -86,7 +86,7 @@ Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sq **Example** -Query with a tuple map: +Query with a tuple: ```sql SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTypeName(res) as type; diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index 71333e6fcce..382306016e6 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -43,7 +43,11 @@ Entries for finished mutations are not deleted right away (the number of preserv For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. -For `ALTER ... ATTACH|DETACH|DROP` queries, you can use the `replication_alter_partitions_sync` setting to set up waiting. Possible values: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all. +For all `ALTER` queries, you can use the [replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync) setting to set up waiting. + +You can specify how long (in seconds) to wait for inactive replicas to execute all `ALTER` queries with the [replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting. + +!!! info "Note" + For all `ALTER` queries, if `replication_alter_partitions_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. For `ALTER TABLE ... UPDATE|DELETE` queries the synchronicity is defined by the [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting. - diff --git a/docs/en/sql-reference/statements/create/function.md b/docs/en/sql-reference/statements/create/function.md new file mode 100644 index 00000000000..ddfcdfef521 --- /dev/null +++ b/docs/en/sql-reference/statements/create/function.md @@ -0,0 +1,59 @@ +--- +toc_priority: 38 +toc_title: FUNCTION +--- + +# CREATE FUNCTION {#create-function} + +Creates a user defined function from a lambda expression. The expression must consist of function parameters, constants, operators, or other function calls. + +**Syntax** + +```sql +CREATE FUNCTION name AS (parameter0, ...) -> expression +``` +A function can have an arbitrary number of parameters. + +There are a few restrictions: + +- The name of a function must be unique among user defined and system functions. +- Recursive functions are not allowed. +- All variables used by a function must be specified in its parameter list. + +If any restriction is violated then an exception is raised. + +**Example** + +Query: + +```sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +SELECT number, linear_equation(number, 2, 1) FROM numbers(3); +``` + +Result: + +``` text +┌─number─┬─plus(multiply(2, number), 1)─┐ +│ 0 │ 1 │ +│ 1 │ 3 │ +│ 2 │ 5 │ +└────────┴──────────────────────────────┘ +``` + +A [conditional function](../../../sql-reference/functions/conditional-functions.md) is called in a user defined function in the following query: + +```sql +CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even'); +SELECT number, parity_str(number) FROM numbers(3); +``` + +Result: + +``` text +┌─number─┬─if(modulo(number, 2), 'odd', 'even')─┐ +│ 0 │ even │ +│ 1 │ odd │ +│ 2 │ even │ +└────────┴──────────────────────────────────────┘ +``` diff --git a/docs/en/sql-reference/statements/create/index.md b/docs/en/sql-reference/statements/create/index.md index 902a4348bac..5721130dd24 100644 --- a/docs/en/sql-reference/statements/create/index.md +++ b/docs/en/sql-reference/statements/create/index.md @@ -12,6 +12,7 @@ Create queries make a new entity of one of the following kinds: - [TABLE](../../../sql-reference/statements/create/table.md) - [VIEW](../../../sql-reference/statements/create/view.md) - [DICTIONARY](../../../sql-reference/statements/create/dictionary.md) +- [FUNCTION](../../../sql-reference/statements/create/function.md) - [USER](../../../sql-reference/statements/create/user.md) - [ROLE](../../../sql-reference/statements/create/role.md) - [ROW POLICY](../../../sql-reference/statements/create/row-policy.md) diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 90a2a46c7cf..552a7b5f1a9 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -97,4 +97,20 @@ Syntax: DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -[Оriginal article](https://clickhouse.tech/docs/en/sql-reference/statements/drop/) +## DROP FUNCTION {#drop-function} + +Deletes a user defined function created by [CREATE FUNCTION](./create/function.md). +System functions can not be dropped. + +**Syntax** + +``` sql +DROP FUNCTION [IF EXISTS] function_name +``` + +**Example** + +``` sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +DROP FUNCTION linear_equation; +``` diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 25dffc36954..2b3cd68fbb2 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -107,11 +107,13 @@ Hierarchy of privileges: - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` + - `CREATE FUNCTION` - [DROP](#grant-drop) - `DROP DATABASE` - `DROP TABLE` - `DROP VIEW` - `DROP DICTIONARY` + - `DROP FUNCTION` - [TRUNCATE](#grant-truncate) - [OPTIMIZE](#grant-optimize) - [SHOW](#grant-show) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 864509cec94..4054f373cc1 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -18,13 +18,17 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I The `OPTMIZE` query is supported for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family, the [MaterializedView](../../engines/table-engines/special/materializedview.md) and the [Buffer](../../engines/table-engines/special/buffer.md) engines. Other table engines aren’t supported. -When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all nodes (if the `replication_alter_partitions_sync` setting is enabled). +When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all replicas (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `2`) or on current replica (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `1`). - If `OPTIMIZE` does not perform a merge for any reason, it does not notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. - If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr). - If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed. - If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. +You can specify how long (in seconds) to wait for inactive replicas to execute `OPTIMIZE` queries by the [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting. + +!!! info "Note" + If the `replication_alter_partitions_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ## BY expression {#by-expression} diff --git a/docs/en/sql-reference/statements/select/where.md b/docs/en/sql-reference/statements/select/where.md index f1532115e55..69505a51db4 100644 --- a/docs/en/sql-reference/statements/select/where.md +++ b/docs/en/sql-reference/statements/select/where.md @@ -6,7 +6,7 @@ toc_title: WHERE `WHERE` clause allows to filter the data that is coming from [FROM](../../../sql-reference/statements/select/from.md) clause of `SELECT`. -If there is a `WHERE` clause, it must contain an expression with the `UInt8` type. This is usually an expression with comparison and logical operators. Rows where this expression evaluates to 0 are expluded from further transformations or result. +If there is a `WHERE` clause, it must contain an expression with the `UInt8` type. This is usually an expression with comparison and logical operators. Rows where this expression evaluates to 0 are excluded from further transformations or result. `WHERE` expression is evaluated on the ability to use indexes and partition pruning, if the underlying table engine supports that. diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index f302a8605e2..b5354196fa4 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -12,3 +12,10 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Removes all data from a table. When the clause `IF EXISTS` is omitted, the query returns an error if the table does not exist. The `TRUNCATE` query is not supported for [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) and [Null](../../engines/table-engines/special/null.md) table engines. + +You can use the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting to set up waiting for actions to be executed on replicas. + +You can specify how long (in seconds) to wait for inactive replicas to execute `TRUNCATE` queries with the [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting. + +!!! info "Note" + If the `replication_alter_partitions_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index d84edb3f46e..ffba8f5c6d3 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -3,7 +3,7 @@ toc_priority: 45 toc_title: s3 --- -# S3 Table Function {#s3-table-function} +# s3 Table Function {#s3-table-function} Provides table-like interface to select/insert files in [Amazon S3](https://aws.amazon.com/s3/). This table function is similar to [hdfs](../../sql-reference/table-functions/hdfs.md), but provides S3-specific features. @@ -125,6 +125,30 @@ INSERT INTO FUNCTION s3('https://storage.yandexcloud.net/my-test-bucket-768/test SELECT name, value FROM existing_table; ``` +## Partitioned Write {#partitioned-write} + +If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency. + +**Examples** + +1. Using partition ID in a key creates separate files: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32') + PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); +``` +As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, and `file_z.csv`. + +2. Using partition ID in a bucket name creates files in different buckets: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); +``` +As a result, the data is written into three files in different buckets: `my_bucket_1/file.csv`, `my_bucket_10/file.csv`, and `my_bucket_20/file.csv`. + **See Also** - [S3 engine](../../engines/table-engines/integrations/s3.md) diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md index 5895bd43d2f..c90b7293e1c 100644 --- a/docs/ru/engines/table-engines/integrations/s3.md +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -151,4 +151,4 @@ ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file- **Смотрите также** -- [Табличная функция S3](../../../sql-reference/table-functions/s3.md) +- [Табличная функция s3](../../../sql-reference/table-functions/s3.md) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 9ad300b8c9c..f55edec62d1 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3308,6 +3308,30 @@ SETTINGS index_granularity = 8192 │ Значение по умолчанию: `0`. +## replication_alter_partitions_sync {#replication-alter-partitions-sync} + +Позволяет настроить ожидание выполнения действий на репликах запросами [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) или [TRUNCATE](../../sql-reference/statements/truncate.md). + +Возможные значения: + +- 0 — не ждать. +- 1 — ждать выполнения действий на своей реплике. +- 2 — ждать выполнения действий на всех репликах. + +Значение по умолчанию: `1`. + +## replication_wait_for_inactive_replica_timeout {#replication-wait-for-inactive-replica-timeout} + +Указывает время ожидания (в секундах) выполнения запросов [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) или [TRUNCATE](../../sql-reference/statements/truncate.md) для неактивных реплик. + +Возможные значения: + +- 0 — не ждать. +- Отрицательное целое число — ждать неограниченное время. +- Положительное целое число — установить соответствующее количество секунд ожидания. + +Значение по умолчанию: `120` секунд. + ## regexp_max_matches_per_row {#regexp-max-matches-per-row} Задает максимальное количество совпадений для регулярного выражения. Настройка применяется для защиты памяти от перегрузки при использовании "жадных" квантификаторов в регулярном выражении для функции [extractAllGroupsHorizontal](../../sql-reference/functions/string-search-functions.md#extractallgroups-horizontal). @@ -3316,4 +3340,4 @@ SETTINGS index_granularity = 8192 │ - Положительное целое число. -Значение по умолчанию: `1000`. \ No newline at end of file +Значение по умолчанию: `1000`. diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 161c1304b7c..694dfef7d75 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -17,13 +17,13 @@ char(number_1, [number_2, ..., number_n]); **Аргументы** -- `number_1, number_2, ..., number_n` — числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql-reference/functions/encoding-functions.md), [Float](../../sql-reference/functions/encoding-functions.md). +- `number_1, number_2, ..., number_n` — числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md). **Возвращаемое значение** - Строка из соответствующих байт. -Тип: `String`. +Тип: [String](../../sql-reference/data-types/string.md). **Пример** @@ -73,61 +73,57 @@ SELECT char(0xE4, 0xBD, 0xA0, 0xE5, 0xA5, 0xBD) AS hello; ## hex {#hex} -Returns a string containing the argument’s hexadecimal representation. +Возвращает строку, содержащую шестнадцатеричное представление аргумента. Синоним: `HEX`. -**Syntax** +**Синтаксис** ``` sql hex(arg) ``` -The function is using uppercase letters `A-F` and not using any prefixes (like `0x`) or suffixes (like `h`). +Функция использует прописные буквы `A-F` и не использует никаких префиксов (например, `0x`) или суффиксов (например, `h`). -For integer arguments, it prints hex digits («nibbles») from the most significant to least significant (big endian or «human readable» order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if leading digit is zero. +Для целочисленных аргументов возвращает шестнадцатеричные цифры от наиболее до наименее значимых (`big endian`, человекочитаемый порядок).Он начинается с самого значимого ненулевого байта (начальные нулевые байты опущены), но всегда выводит обе цифры каждого байта, даже если начальная цифра равна нулю. -Example: +Значения типа [Date](../../sql-reference/data-types/date.md) и [DateTime](../../sql-reference/data-types/datetime.md) формируются как соответствующие целые числа (количество дней с момента Unix-эпохи для `Date` и значение Unix Timestamp для `DateTime`). -**Example** +Для [String](../../sql-reference/data-types/string.md) и [FixedString](../../sql-reference/data-types/fixedstring.md), все байты просто кодируются как два шестнадцатеричных числа. Нулевые байты не опущены. -Query: +Значения [Float](../../sql-reference/data-types/float.md) и [Decimal](../../sql-reference/data-types/decimal.md) кодируются как их представление в памяти. Поскольку ClickHouse поддерживает архитектуру `little-endian`, они кодируются от младшего к старшему байту. Нулевые начальные/конечные байты не опущены. + +**Аргументы** + +- `arg` — значение для преобразования в шестнадцатеричное. [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +- Строка — шестнадцатеричное представление аргумента. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Примеры** + +Запрос: ``` sql SELECT hex(1); ``` -Result: +Результат: ``` text 01 ``` -Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). - -For `String` and `FixedString`, all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted. - -Values of floating point and Decimal types are encoded as their representation in memory. As we support little endian architecture, they are encoded in little endian. Zero leading/trailing bytes are not omitted. - -**Parameters** - -- `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/functions/encoding-functions.md), [UInt](../../sql-reference/functions/encoding-functions.md), [Float](../../sql-reference/functions/encoding-functions.md), [Decimal](../../sql-reference/functions/encoding-functions.md), [Date](../../sql-reference/functions/encoding-functions.md) or [DateTime](../../sql-reference/functions/encoding-functions.md). - -**Returned value** - -- A string with the hexadecimal representation of the argument. - -Type: `String`. - -**Example** - -Query: +Запрос: ``` sql -SELECT hex(toFloat32(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat32(number)) AS hex_presentation FROM numbers(15, 2); ``` -Result: +Результат: ``` text ┌─hex_presentation─┐ @@ -136,13 +132,13 @@ Result: └──────────────────┘ ``` -Query: +Запрос: ``` sql -SELECT hex(toFloat64(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat64(number)) AS hex_presentation FROM numbers(15, 2); ``` -Result: +Результат: ``` text ┌─hex_presentation─┐ @@ -208,6 +204,141 @@ SELECT reinterpretAsUInt64(reverse(unhex('FFF'))) AS num; └──────┘ ``` +## bin {#bin} + +Возвращает строку, содержащую бинарное представление аргумента. + +**Синтаксис** + +``` sql +bin(arg) +``` + +Синоним: `BIN`. + +Для целочисленных аргументов возвращаются двоичные числа от наиболее значимого до наименее значимого (`big-endian`, человекочитаемый порядок). Порядок начинается с самого значимого ненулевого байта (начальные нулевые байты опущены), но всегда возвращает восемь цифр каждого байта, если начальная цифра равна нулю. + +Значения типа [Date](../../sql-reference/data-types/date.md) и [DateTime](../../sql-reference/data-types/datetime.md) формируются как соответствующие целые числа (количество дней с момента Unix-эпохи для `Date` и значение Unix Timestamp для `DateTime`). + +Для [String](../../sql-reference/data-types/string.md) и [FixedString](../../sql-reference/data-types/fixedstring.md) все байты кодируются как восемь двоичных чисел. Нулевые байты не опущены. + +Значения [Float](../../sql-reference/data-types/float.md) и [Decimal](../../sql-reference/data-types/decimal.md) кодируются как их представление в памяти. Поскольку ClickHouse поддерживает архитектуру `little-endian`, они кодируются от младшего к старшему байту. Нулевые начальные/конечные байты не опущены. + +**Аргументы** + +- `arg` — значение для преобразования в двоичный код. [String](../../sql-reference/data-types/string.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +- Бинарная строка (BLOB) — двоичное представление аргумента. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Примеры** + +Запрос: + +``` sql +SELECT bin(14); +``` + +Результат: + +``` text +┌─bin(14)──┐ +│ 00001110 │ +└──────────┘ +``` + +Запрос: + +``` sql +SELECT bin(toFloat32(number)) AS bin_presentation FROM numbers(15, 2); +``` + +Результат: + +``` text +┌─bin_presentation─────────────────┐ +│ 00000000000000000111000001000001 │ +│ 00000000000000001000000001000001 │ +└──────────────────────────────────┘ +``` + +Запрос: + +``` sql +SELECT bin(toFloat64(number)) AS bin_presentation FROM numbers(15, 2); +``` + +Результат: + +``` text +┌─bin_presentation─────────────────────────────────────────────────┐ +│ 0000000000000000000000000000000000000000000000000010111001000000 │ +│ 0000000000000000000000000000000000000000000000000011000001000000 │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## unbin {#unbinstr} + +Интерпретирует каждую пару двоичных цифр аргумента как число и преобразует его в байт, представленный числом. Функция выполняет операцию, противоположную [bin](#bin). + +**Синтаксис** + +``` sql +unbin(arg) +``` + +Синоним: `UNBIN`. + +Для числового аргумента `unbin()` не возвращает значение, обратное результату `bin()`. Чтобы преобразовать результат в число, используйте функции [reverse](../../sql-reference/functions/string-functions.md#reverse) и [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#reinterpretasuint8163264). + +!!! note "Примечание" + Если `unbin` вызывается из клиента `clickhouse-client`, бинарная строка возвращается в кодировке UTF-8. + +Поддерживает двоичные цифры `0` и `1`. Количество двоичных цифр не обязательно должно быть кратно восьми. Если строка аргумента содержит что-либо, кроме двоичных цифр, возвращается некоторый результат, определенный реализацией (ошибки не возникает). + +**Аргументы** + +- `arg` — строка, содержащая любое количество двоичных цифр. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Бинарная строка (BLOB). + +Тип: [String](../../sql-reference/data-types/string.md). + +**Примеры** + +Запрос: + +``` sql +SELECT UNBIN('001100000011000100110010'), UNBIN('0100110101111001010100110101000101001100'); +``` + +Результат: + +``` text +┌─unbin('001100000011000100110010')─┬─unbin('0100110101111001010100110101000101001100')─┐ +│ 012 │ MySQL │ +└───────────────────────────────────┴───────────────────────────────────────────────────┘ +``` + +Запрос: + +``` sql +SELECT reinterpretAsUInt64(reverse(unbin('1110'))) AS num; +``` + +Результат: + +``` text +┌─num─┐ +│ 14 │ +└─────┘ +``` + ## UUIDStringToNum(str) {#uuidstringtonumstr} Принимает строку, содержащую 36 символов в формате `123e4567-e89b-12d3-a456-426655440000`, и возвращает в виде набора байт в FixedString(16). @@ -263,7 +394,7 @@ SELECT bitPositionsToArray(toInt8(1)) AS bit_positions; Запрос: ``` sql -select bitPositionsToArray(toInt8(-1)) as bit_positions; +SELECT bitPositionsToArray(toInt8(-1)) AS bit_positions; ``` Результат: diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index 15da9d36ef5..92bd1c1c2f8 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -58,6 +58,10 @@ str -> str != Referer Для некоторых функций первый аргумент (лямбда-функция) может отсутствовать. В этом случае подразумевается тождественное отображение. +## Пользовательские функции {#user-defined-functions} + +Функции можно создавать с помощью выражения [CREATE FUNCTION](../statements/create/function.md). Для удаления таких функций используется выражение [DROP FUNCTION](../statements/drop.md#drop-function). + ## Обработка ошибок {#obrabotka-oshibok} Некоторые функции могут кидать исключения в случае ошибочных данных. В этом случае, выполнение запроса прерывается, и текст ошибки выводится клиенту. При распределённой обработке запроса, при возникновении исключения на одном из серверов, на другие серверы пытается отправиться просьба тоже прервать выполнение запроса. diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index 4775152fb54..e4cc1fefab4 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -73,22 +73,22 @@ SELECT a['key2'] FROM table_map; **Синтаксис** ``` sql -mapAdd(Tuple(Array, Array), Tuple(Array, Array) [, ...]) +mapAdd(arg1, arg2 [, ...]) ``` **Аргументы** -Аргументами являются [кортежи](../../sql-reference/data-types/tuple.md#tuplet1-t2) из двух [массивов](../../sql-reference/data-types/array.md#data-type-array), где элементы в первом массиве представляют ключи, а второй массив содержит значения для каждого ключа. +Аргументами являются контейнеры [Map](../../sql-reference/data-types/map.md) или [кортежи](../../sql-reference/data-types/tuple.md#tuplet1-t2) из двух [массивов](../../sql-reference/data-types/array.md#data-type-array), где элементы в первом массиве представляют ключи, а второй массив содержит значения для каждого ключа. Все массивы ключей должны иметь один и тот же тип, а все массивы значений должны содержать элементы, которые можно приводить к одному типу ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) или [Float64](../../sql-reference/data-types/float.md#float32-float64)). Общий приведенный тип используется в качестве типа для результирующего массива. **Возвращаемое значение** -- Возвращает один [кортеж](../../sql-reference/data-types/tuple.md#tuplet1-t2), в котором первый массив содержит отсортированные ключи, а второй — значения. +- В зависимости от типа аргументов возвращает один [Map](../../sql-reference/data-types/map.md) или [кортеж](../../sql-reference/data-types/tuple.md#tuplet1-t2), в котором первый массив содержит отсортированные ключи, а второй — значения. **Пример** -Запрос: +Запрос с кортежем: ``` sql SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTypeName(res) as type; @@ -102,6 +102,20 @@ SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTy └───────────────┴────────────────────────────────────┘ ``` +Запрос с контейнером `Map`: + +```sql +SELECT mapAdd(map(1,1), map(1,1)); +``` + +Result: + +```text +┌─mapAdd(map(1, 1), map(1, 1))─┐ +│ {1:2} │ +└──────────────────────────────┘ +``` + ## mapSubtract {#function-mapsubtract} Собирает все ключи и вычитает соответствующие значения. diff --git a/docs/ru/sql-reference/statements/alter/index.md b/docs/ru/sql-reference/statements/alter/index.md index 043ac3839d9..2b7caa5ad5b 100644 --- a/docs/ru/sql-reference/statements/alter/index.md +++ b/docs/ru/sql-reference/statements/alter/index.md @@ -64,8 +64,11 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Для нереплицируемых таблиц, все запросы `ALTER` выполняются синхронно. Для реплицируемых таблиц, запрос всего лишь добавляет инструкцию по соответствующим действиям в `ZooKeeper`, а сами действия осуществляются при первой возможности. Но при этом, запрос может ждать завершения выполнения этих действий на всех репликах. -Для запросов `ALTER ... ATTACH|DETACH|DROP` можно настроить ожидание, с помощью настройки `replication_alter_partitions_sync`. -Возможные значения: `0` - не ждать, `1` - ждать выполнения только у себя (по умолчанию), `2` - ждать всех. +Для всех запросов `ALTER` можно настроить ожидание с помощью настройки [replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync). + +Вы можете указать время ожидания (в секундах) выполнения всех запросов `ALTER` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout). + +!!! info "Примечание" + Для всех запросов `ALTER` при `replication_alter_partitions_sync = 2` и неактивности некоторых реплик больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, генерируется исключение `UNFINISHED`. Для запросов `ALTER TABLE ... UPDATE|DELETE` синхронность выполнения определяется настройкой [mutations_sync](../../../operations/settings/settings.md#mutations_sync). - diff --git a/docs/ru/sql-reference/statements/create/function.md b/docs/ru/sql-reference/statements/create/function.md new file mode 100644 index 00000000000..90838b25744 --- /dev/null +++ b/docs/ru/sql-reference/statements/create/function.md @@ -0,0 +1,59 @@ +--- +toc_priority: 38 +toc_title: FUNCTION +--- + +# CREATE FUNCTION {#create-function} + +Создает пользовательскую функцию из лямбда-выражения. Выражение должно состоять из параметров функции, констант, операторов и вызовов других функций. + +**Синтаксис** + +```sql +CREATE FUNCTION name AS (parameter0, ...) -> expression +``` +У функции может быть произвольное число параметров. + +Существует несколько ограничений на создаваемые функции: + +- Имя функции должно быть уникальным среди всех пользовательских и системных функций. +- Рекурсивные функции запрещены. +- Все переменные, используемые функцией, должны быть перечислены в списке ее параметров. + +Если какое-нибудь ограничение нарушается, то при попытке создать функцию возникает исключение. + +**Пример** + +Запрос: + +```sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +SELECT number, linear_equation(number, 2, 1) FROM numbers(3); +``` + +Результат: + +``` text +┌─number─┬─plus(multiply(2, number), 1)─┐ +│ 0 │ 1 │ +│ 1 │ 3 │ +│ 2 │ 5 │ +└────────┴──────────────────────────────┘ +``` + +В следующем запросе пользовательская функция вызывает [условную функцию](../../../sql-reference/functions/conditional-functions.md): + +```sql +CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even'); +SELECT number, parity_str(number) FROM numbers(3); +``` + +Результат: + +``` text +┌─number─┬─if(modulo(number, 2), 'odd', 'even')─┐ +│ 0 │ even │ +│ 1 │ odd │ +│ 2 │ even │ +└────────┴──────────────────────────────────────┘ +``` diff --git a/docs/ru/sql-reference/statements/create/index.md b/docs/ru/sql-reference/statements/create/index.md index dfa5c28fff7..61d4d053fec 100644 --- a/docs/ru/sql-reference/statements/create/index.md +++ b/docs/ru/sql-reference/statements/create/index.md @@ -12,6 +12,7 @@ toc_title: "Обзор" - [TABLE](../../../sql-reference/statements/create/table.md) - [VIEW](../../../sql-reference/statements/create/view.md) - [DICTIONARY](../../../sql-reference/statements/create/dictionary.md) +- [FUNCTION](../../../sql-reference/statements/create/function.md) - [USER](../../../sql-reference/statements/create/user.md) - [ROLE](../../../sql-reference/statements/create/role.md) - [ROW POLICY](../../../sql-reference/statements/create/row-policy.md) diff --git a/docs/ru/sql-reference/statements/drop.md b/docs/ru/sql-reference/statements/drop.md index 118f8eb923a..437c2d02a94 100644 --- a/docs/ru/sql-reference/statements/drop.md +++ b/docs/ru/sql-reference/statements/drop.md @@ -97,3 +97,20 @@ DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` +## DROP FUNCTION {#drop-function} + +Удаляет пользовательскую функцию, созданную с помощью [CREATE FUNCTION](./create/function.md). +Удалить системные функции нельзя. + +**Синтаксис** + +``` sql +DROP FUNCTION [IF EXISTS] function_name +``` + +**Пример** + +``` sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +DROP FUNCTION linear_equation; +``` diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 8d6605e1571..45ba9bb0343 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -109,11 +109,13 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` + - `CREATE FUNCTION` - [DROP](#grant-drop) - `DROP DATABASE` - `DROP TABLE` - `DROP VIEW` - `DROP DICTIONARY` + - `DROP FUNCTION` - [TRUNCATE](#grant-truncate) - [OPTIMIZE](#grant-optimize) - [SHOW](#grant-show) diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index 1f0c5a0ebe9..e6a71c4f611 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -18,7 +18,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I Может применяться к таблицам семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md), [MaterializedView](../../engines/table-engines/special/materializedview.md) и [Buffer](../../engines/table-engines/special/buffer.md). Другие движки таблиц не поддерживаются. -Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`). +Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех репликах (если значение настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) равно `2`) или на текущей реплике (если значение настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) равно `1`). - По умолчанию, если запросу `OPTIMIZE` не удалось выполнить слияние, то ClickHouse не оповещает клиента. Чтобы включить оповещения, используйте настройку [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop). @@ -26,6 +26,11 @@ ClickHouse не оповещает клиента. Чтобы включить - Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске данных. Кроме того, слияние является принудительным, даже если выполняются параллельные слияния. - Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех столбцах), имеет смысл только для движка MergeTree. +Вы можете указать время ожидания (в секундах) выполнения запросов `OPTIMIZE` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout). + +!!! info "Примечание" + Если значение настройки `replication_alter_partitions_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`. + ## Выражение BY {#by-expression} Чтобы выполнить дедупликацию по произвольному набору столбцов, вы можете явно указать список столбцов или использовать любую комбинацию подстановки [`*`](../../sql-reference/statements/select/index.md#asterisk), выражений [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) и [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier). diff --git a/docs/ru/sql-reference/statements/truncate.md b/docs/ru/sql-reference/statements/truncate.md index 63f7fa86ea5..028959690cd 100644 --- a/docs/ru/sql-reference/statements/truncate.md +++ b/docs/ru/sql-reference/statements/truncate.md @@ -13,4 +13,9 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Запрос `TRUNCATE` не поддерживается для следующих движков: [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) и [Null](../../engines/table-engines/special/null.md). +Вы можете настроить ожидание выполнения действий на репликах с помощью настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync). +Вы можете указать время ожидания (в секундах) выполнения запросов `TRUNCATE` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout). + +!!! info "Примечание" + Если значение настройки `replication_alter_partitions_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`. diff --git a/docs/ru/sql-reference/table-functions/s3.md b/docs/ru/sql-reference/table-functions/s3.md index 597f145c096..c8dbcf81559 100644 --- a/docs/ru/sql-reference/table-functions/s3.md +++ b/docs/ru/sql-reference/table-functions/s3.md @@ -133,6 +133,30 @@ INSERT INTO FUNCTION s3('https://storage.yandexcloud.net/my-test-bucket-768/test SELECT name, value FROM existing_table; ``` +## Партиционирование при записи данных {#partitioned-write} + +Если при добавлении данных в таблицу S3 указать выражение `PARTITION BY`, то для каждого значения ключа партиционирования создается отдельный файл. Это повышает эффективность операций чтения. + +**Примеры** + +1. При использовании ID партиции в имени ключа создаются отдельные файлы: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); +``` +В результате данные будут записаны в три файла: `file_x.csv`, `file_y.csv` и `file_z.csv`. + +2. При использовании ID партиции в названии бакета создаются файлы в разных бакетах: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); +``` +В результате будут созданы три файла в разных бакетах: `my_bucket_1/file.csv`, `my_bucket_10/file.csv` и `my_bucket_20/file.csv`. + **Смотрите также** - [Движок таблиц S3](../../engines/table-engines/integrations/s3.md) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 87ae03161a9..c594f01861b 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1031,19 +1031,30 @@ private: if (server_exception) { bool print_stack_trace = config().getBool("stacktrace", false); - std::cerr << "Received exception from server (version " << server_version << "):" << std::endl - << getExceptionMessage(*server_exception, print_stack_trace, true) << std::endl; + fmt::print(stderr, "Received exception from server (version {}):\n{}\n", + server_version, + getExceptionMessage(*server_exception, print_stack_trace, true)); if (is_interactive) - std::cerr << std::endl; + { + fmt::print(stderr, "\n"); + } + else + { + fmt::print(stderr, "(query: {})\n", full_query); + } } if (client_exception) { - fmt::print(stderr, "Error on processing query '{}':\n{}\n", full_query, client_exception->message()); + fmt::print(stderr, "Error on processing query: {}\n", client_exception->message()); if (is_interactive) { fmt::print(stderr, "\n"); } + else + { + fmt::print(stderr, "(query: {})\n", full_query); + } } // A debug check -- at least some exception must be set, if the error @@ -1244,13 +1255,17 @@ private: if (!server_exception) { error_matches_hint = false; - fmt::print(stderr, "Expected server error code '{}' but got no server error.\n", test_hint.serverError()); + fmt::print(stderr, "Expected server error code '{}' but got no server error (query: {}).\n", + test_hint.serverError(), + full_query); } else if (server_exception->code() != test_hint.serverError()) { error_matches_hint = false; - std::cerr << "Expected server error code: " << test_hint.serverError() << " but got: " << server_exception->code() - << "." << std::endl; + fmt::print(stderr, "Expected server error code: {} but got: {} (query: {}).\n", + test_hint.serverError(), + server_exception->code(), + full_query); } } @@ -1259,13 +1274,17 @@ private: if (!client_exception) { error_matches_hint = false; - fmt::print(stderr, "Expected client error code '{}' but got no client error.\n", test_hint.clientError()); + fmt::print(stderr, "Expected client error code '{}' but got no client error (query: {}).\n", + test_hint.clientError(), + full_query); } else if (client_exception->code() != test_hint.clientError()) { error_matches_hint = false; - fmt::print( - stderr, "Expected client error code '{}' but got '{}'.\n", test_hint.clientError(), client_exception->code()); + fmt::print(stderr, "Expected client error code '{}' but got '{}' (query: {}).\n", + test_hint.clientError(), + client_exception->code(), + full_query); } } @@ -1281,13 +1300,17 @@ private: { if (test_hint.clientError()) { - fmt::print(stderr, "The query succeeded but the client error '{}' was expected.\n", test_hint.clientError()); + fmt::print(stderr, "The query succeeded but the client error '{}' was expected (query: {}).\n", + test_hint.clientError(), + full_query); error_matches_hint = false; } if (test_hint.serverError()) { - fmt::print(stderr, "The query succeeded but the server error '{}' was expected.\n", test_hint.serverError()); + fmt::print(stderr, "The query succeeded but the server error '{}' was expected (query: {}).\n", + test_hint.serverError(), + full_query); error_matches_hint = false; } } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 4a6d1e206e7..0c834174519 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -962,7 +962,7 @@ if (ThreadFuzzer::instance().isEffective()) global_context->setMMappedFileCache(mmap_cache_size); #if USE_EMBEDDED_COMPILER - constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 1024; + constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 128; size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", compiled_expression_cache_size_default); CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_size); #endif diff --git a/programs/server/config.xml b/programs/server/config.xml index 32207e2b6b3..b8ef17458be 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -331,7 +331,7 @@ 1000 - 1073741824 + 134217728 /var/lib/clickhouse/ diff --git a/programs/server/config.yaml.example b/programs/server/config.yaml.example index 5b2da1d3128..ae4eac49a64 100644 --- a/programs/server/config.yaml.example +++ b/programs/server/config.yaml.example @@ -280,7 +280,7 @@ mark_cache_size: 5368709120 mmap_cache_size: 1000 # Cache size for compiled expressions. -compiled_expression_cache_size: 1073741824 +compiled_expression_cache_size: 134217728 # Path to data directory, with trailing slash. path: /var/lib/clickhouse/ diff --git a/src/AggregateFunctions/AggregateFunctionArray.cpp b/src/AggregateFunctions/AggregateFunctionArray.cpp index 982180ab50c..3591bea5f9e 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionArray.cpp @@ -21,6 +21,8 @@ class AggregateFunctionCombinatorArray final : public IAggregateFunctionCombinat public: String getName() const override { return "Array"; } + bool supportsNesting() const override { return true; } + DataTypes transformArguments(const DataTypes & arguments) const override { if (arguments.empty()) diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index c9dcdb54424..c9a44dba6f2 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -29,6 +29,7 @@ namespace ErrorCodes { extern const int UNKNOWN_AGGREGATE_FUNCTION; extern const int LOGICAL_ERROR; + extern const int ILLEGAL_AGGREGATION; } const String & getAggregateFunctionCanonicalNameIfAny(const String & name) @@ -159,13 +160,32 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name)) { + const std::string & combinator_name = combinator->getName(); + if (combinator->isForInternalUsageOnly()) - throw Exception("Aggregate function combinator '" + combinator->getName() + "' is only for internal usage", ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION); + throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, + "Aggregate function combinator '{}' is only for internal usage", + combinator_name); if (query_context && query_context->getSettingsRef().log_queries) - query_context->addQueryFactoriesInfo(Context::QueryLogFactories::AggregateFunctionCombinator, combinator->getName()); + query_context->addQueryFactoriesInfo(Context::QueryLogFactories::AggregateFunctionCombinator, combinator_name); + + String nested_name = name.substr(0, name.size() - combinator_name.size()); + /// Nested identical combinators (i.e. uniqCombinedIfIf) is not + /// supported (since they even don't work -- silently). + /// + /// But non-identical does supported and works, for example + /// uniqCombinedIfMergeIf, it is useful in case when the underlying + /// storage stores AggregateFunction(uniqCombinedIf) and in SELECT you + /// need to filter aggregation result based on another column for + /// example. + if (!combinator->supportsNesting() && nested_name.ends_with(combinator_name)) + { + throw Exception(ErrorCodes::ILLEGAL_AGGREGATION, + "Nested identical combinator '{}' is not supported", + combinator_name); + } - String nested_name = name.substr(0, name.size() - combinator->getName().size()); DataTypes nested_types = combinator->transformArguments(argument_types); Array nested_parameters = combinator->transformParameters(parameters); diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index 5082952f386..4ac6a2dce21 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -10,7 +10,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_AGGREGATION; } class AggregateFunctionCombinatorIf final : public IAggregateFunctionCombinator @@ -37,10 +36,6 @@ public: const DataTypes & arguments, const Array & params) const override { - if (nested_function->getName().find(getName()) != String::npos) - { - throw Exception(ErrorCodes::ILLEGAL_AGGREGATION, "nested function for {0}-combinator must not have {0}-combinator", getName()); - } return std::make_shared(nested_function, arguments, params); } }; diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.cpp b/src/AggregateFunctions/AggregateFunctionOrFill.cpp index 3ba20e65e32..7aee0289879 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.cpp +++ b/src/AggregateFunctions/AggregateFunctionOrFill.cpp @@ -23,6 +23,9 @@ private: public: explicit AggregateFunctionCombinatorOrFill(Kind kind_) : kind(kind_) {} + /// Due to aggregate_functions_null_for_empty + bool supportsNesting() const override { return true; } + String getName() const override { return kind == Kind::OrNull ? "OrNull" : "OrDefault"; diff --git a/src/AggregateFunctions/IAggregateFunctionCombinator.h b/src/AggregateFunctions/IAggregateFunctionCombinator.h index f9953c83a95..37fcfe42c10 100644 --- a/src/AggregateFunctions/IAggregateFunctionCombinator.h +++ b/src/AggregateFunctions/IAggregateFunctionCombinator.h @@ -35,6 +35,10 @@ public: virtual bool isForInternalUsageOnly() const { return false; } + /** Does combinator supports nesting (of itself, i.e. ArrayArray or IfIf) + */ + virtual bool supportsNesting() const { return false; } + /** From the arguments for combined function (ex: UInt64, UInt8 for sumIf), * get the arguments for nested function (ex: UInt64 for sum). * If arguments are not suitable for combined function, throw an exception. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c7c555c2f3b..7cc8433e153 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -261,7 +261,7 @@ dbms_target_include_directories (PUBLIC "${ClickHouse_SOURCE_DIR}/src" "${ClickH target_include_directories (clickhouse_common_io PUBLIC "${ClickHouse_SOURCE_DIR}/src" "${ClickHouse_BINARY_DIR}/src") if (USE_EMBEDDED_COMPILER) - dbms_target_link_libraries (PRIVATE ${REQUIRED_LLVM_LIBRARIES}) + dbms_target_link_libraries (PUBLIC ${REQUIRED_LLVM_LIBRARIES}) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () diff --git a/src/Columns/MaskOperations.cpp b/src/Columns/MaskOperations.cpp index 759d0af7127..b63f2d25665 100644 --- a/src/Columns/MaskOperations.cpp +++ b/src/Columns/MaskOperations.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace DB @@ -177,19 +178,21 @@ MaskInfo extractMaskFromConstOrNull( template MaskInfo extractMaskImpl( PaddedPODArray & mask, - const ColumnPtr & column, + const ColumnPtr & col, UInt8 null_value, const PaddedPODArray * null_bytemap, PaddedPODArray * nulls = nullptr) { + auto column = col->convertToFullColumnIfLowCardinality(); + /// Special implementation for Null and Const columns. if (column->onlyNull() || checkAndGetColumn(*column)) return extractMaskFromConstOrNull(mask, column, null_value, nulls); - if (const auto * col = checkAndGetColumn(*column)) + if (const auto * nullable_column = checkAndGetColumn(*column)) { - const PaddedPODArray & null_map = col->getNullMapData(); - return extractMaskImpl(mask, col->getNestedColumnPtr(), null_value, &null_map, nulls); + const PaddedPODArray & null_map = nullable_column->getNullMapData(); + return extractMaskImpl(mask, nullable_column->getNestedColumnPtr(), null_value, &null_map, nulls); } MaskInfo mask_info; @@ -314,3 +317,4 @@ void copyMask(const PaddedPODArray & from, PaddedPODArray & to) } } + diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 812f888b284..a9d4700490e 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -417,11 +417,7 @@ void StackTrace::toStringEveryLine(std::function call std::string StackTrace::toString() const { - /// Calculation of stack trace text is extremely slow. - /// We use simple cache because otherwise the server could be overloaded by trash queries. - - static SimpleCache func_cached; - return func_cached(frame_pointers, offset, size); + return toStringStatic(frame_pointers, offset, size); } std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t size) @@ -432,6 +428,23 @@ std::string StackTrace::toString(void ** frame_pointers_, size_t offset, size_t for (size_t i = 0; i < size; ++i) frame_pointers_copy[i] = frame_pointers_[i]; - static SimpleCache func_cached; - return func_cached(frame_pointers_copy, offset, size); + return toStringStatic(frame_pointers_copy, offset, size); +} + +static SimpleCache & cacheInstance() +{ + static SimpleCache cache; + return cache; +} + +std::string StackTrace::toStringStatic(const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size) +{ + /// Calculation of stack trace text is extremely slow. + /// We use simple cache because otherwise the server could be overloaded by trash queries. + return cacheInstance()(frame_pointers, offset, size); +} + +void StackTrace::dropCache() +{ + cacheInstance().drop(); } diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 8f8c88c29fe..62acb70563f 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -61,6 +61,8 @@ public: std::string toString() const; static std::string toString(void ** frame_pointers, size_t offset, size_t size); + static std::string toStringStatic(const FramePointers & frame_pointers, size_t offset, size_t size); + static void dropCache(); static void symbolize(const FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); void toStringEveryLine(std::function callback) const; diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index a23184c9c0a..2d875b7042d 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -462,12 +462,22 @@ String SymbolIndex::getBuildIDHex() const return build_id_hex; } -MultiVersion::Version SymbolIndex::instance(bool reload) +MultiVersion & SymbolIndex::instanceImpl() { static MultiVersion instance(std::unique_ptr(new SymbolIndex)); - if (reload) - instance.set(std::unique_ptr(new SymbolIndex)); - return instance.get(); + return instance; +} + +MultiVersion::Version SymbolIndex::instance() +{ + return instanceImpl().get(); +} + +void SymbolIndex::reload() +{ + instanceImpl().set(std::unique_ptr(new SymbolIndex)); + /// Also drop stacktrace cache. + StackTrace::dropCache(); } } diff --git a/src/Common/SymbolIndex.h b/src/Common/SymbolIndex.h index 65e446a7fc4..37862987bd2 100644 --- a/src/Common/SymbolIndex.h +++ b/src/Common/SymbolIndex.h @@ -22,7 +22,8 @@ protected: SymbolIndex() { update(); } public: - static MultiVersion::Version instance(bool reload = false); + static MultiVersion::Version instance(); + static void reload(); struct Symbol { @@ -60,6 +61,7 @@ private: Data data; void update(); + static MultiVersion & instanceImpl(); }; } diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index f2b6ed10a39..5716d02803c 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -157,14 +158,16 @@ public: class ThreadFromGlobalPool { public: - ThreadFromGlobalPool() {} + ThreadFromGlobalPool() = default; template explicit ThreadFromGlobalPool(Function && func, Args &&... args) : state(std::make_shared()) + , thread_id(std::make_shared()) { /// NOTE: If this will throw an exception, the destructor won't be called. GlobalThreadPool::instance().scheduleOrThrow([ + thread_id = thread_id, state = state, func = std::forward(func), args = std::make_tuple(std::forward(args)...)]() mutable /// mutable is needed to destroy capture @@ -172,6 +175,8 @@ public: auto event = std::move(state); SCOPE_EXIT(event->set()); + thread_id = std::make_shared(std::this_thread::get_id()); + /// This moves are needed to destroy function and arguments before exit. /// It will guarantee that after ThreadFromGlobalPool::join all captured params are destroyed. auto function = std::move(func); @@ -194,6 +199,7 @@ public: if (joinable()) abort(); state = std::move(rhs.state); + thread_id = std::move(rhs.thread_id); return *this; } @@ -221,12 +227,18 @@ public: bool joinable() const { - return state != nullptr; + if (!state) + return false; + /// Thread cannot join itself. + if (*thread_id == std::this_thread::get_id()) + return false; + return true; } private: /// The state used in this object and inside the thread job. std::shared_ptr state; + std::shared_ptr thread_id; }; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f331ad2d7d3..325e6ea58c5 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -499,7 +499,7 @@ class IColumn; M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ \ M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \ - M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable', 'disable', 'force_enable'", 0) \ + M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \ \ M(String, local_filesystem_read_method, "pread", "Method of reading data from local filesystem, one of: read, pread, mmap, pread_threadpool.", 0) \ M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \ diff --git a/src/DataStreams/ShellCommandSource.h b/src/DataStreams/ShellCommandSource.h index befdbc796ba..21d0acaf81a 100644 --- a/src/DataStreams/ShellCommandSource.h +++ b/src/DataStreams/ShellCommandSource.h @@ -52,14 +52,12 @@ public: const std::string & format, const Block & sample_block, std::unique_ptr && command_, - Poco::Logger * log_, std::vector && send_data_tasks = {}, const ShellCommandSourceConfiguration & configuration_ = {}, std::shared_ptr process_pool_ = nullptr) : SourceWithProgress(sample_block) , command(std::move(command_)) , configuration(configuration_) - , log(log_) , process_pool(process_pool_) { for (auto && send_data_task : send_data_tasks) @@ -133,7 +131,6 @@ protected: } catch (...) { - tryLogCurrentException(log); command = nullptr; throw; } @@ -176,8 +173,6 @@ private: size_t current_read_rows = 0; - Poco::Logger * log; - std::shared_ptr process_pool; QueryPipeline pipeline; diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 560d2d716c9..5a51704e98b 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -233,7 +233,8 @@ void MaterializedMySQLSyncThread::stopSynchronization() if (!sync_quit && background_thread_pool) { sync_quit = true; - background_thread_pool->join(); + if (background_thread_pool->joinable()) + background_thread_pool->join(); client.disconnect(); } } diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index a274e820e65..0a6708416a3 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -72,7 +72,7 @@ Pipe ExecutableDictionarySource::loadAll() ShellCommand::Config config(configuration.command); auto process = ShellCommand::execute(config); - Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process), log)); + Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process))); return pipe; } @@ -91,7 +91,7 @@ Pipe ExecutableDictionarySource::loadUpdatedAll() LOG_TRACE(log, "loadUpdatedAll {}", command_with_update_field); ShellCommand::Config config(command_with_update_field); auto process = ShellCommand::execute(config); - Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process), log)); + Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process))); return pipe; } @@ -120,13 +120,20 @@ Pipe ExecutableDictionarySource::getStreamForBlock(const Block & block) ShellCommandSource::SendDataTask task = {[process_in, block, this]() { auto & out = *process_in; + + if (configuration.send_chunk_header) + { + writeText(block.rows(), out); + writeChar('\n', out); + } + auto output_stream = context->getOutputStream(configuration.format, out, block.cloneEmpty()); formatBlock(output_stream, block); out.close(); }}; std::vector tasks = {std::move(task)}; - Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process), log, std::move(tasks))); + Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process), std::move(tasks))); if (configuration.implicit_key) pipe.addTransform(std::make_shared(block, pipe.getHeader())); @@ -188,7 +195,8 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) .format = config.getString(settings_config_prefix + ".format"), .update_field = config.getString(settings_config_prefix + ".update_field", ""), .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1), - .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false) + .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false), + .send_chunk_header = config.getBool(settings_config_prefix + ".send_chunk_header", false) }; return std::make_unique(dict_struct, configuration, sample_block, context); diff --git a/src/Dictionaries/ExecutableDictionarySource.h b/src/Dictionaries/ExecutableDictionarySource.h index 3133bc12b09..5b20deceb28 100644 --- a/src/Dictionaries/ExecutableDictionarySource.h +++ b/src/Dictionaries/ExecutableDictionarySource.h @@ -19,13 +19,15 @@ public: struct Configuration { - const std::string command; - const std::string format; - const std::string update_field; - const UInt64 update_lag; + std::string command; + std::string format; + std::string update_field; + UInt64 update_lag; /// Implicit key means that the source script will return only values, /// and the correspondence to the requested keys is determined implicitly - by the order of rows in the result. - const bool implicit_key; + bool implicit_key; + /// Send number_of_rows\n before sending chunk to process + bool send_chunk_header; }; ExecutableDictionarySource( diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index c5d081bb3e6..c5ae4803fd4 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -100,7 +100,7 @@ Pipe ExecutablePoolDictionarySource::getStreamForBlock(const Block & block) config.terminate_in_destructor_strategy = ShellCommand::DestructorStrategy{ true /*terminate_in_destructor*/, configuration.command_termination_timeout }; auto shell_command = ShellCommand::execute(config); return shell_command; - }, configuration.max_command_execution_time * 10000); + }, configuration.max_command_execution_time * 1000); if (!result) throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, @@ -112,6 +112,13 @@ Pipe ExecutablePoolDictionarySource::getStreamForBlock(const Block & block) ShellCommandSource::SendDataTask task = [process_in, block, this]() mutable { auto & out = *process_in; + + if (configuration.send_chunk_header) + { + writeText(block.rows(), out); + writeChar('\n', out); + } + auto output_stream = context->getOutputStream(configuration.format, out, block.cloneEmpty()); formatBlock(output_stream, block); }; @@ -120,7 +127,7 @@ Pipe ExecutablePoolDictionarySource::getStreamForBlock(const Block & block) ShellCommandSourceConfiguration command_configuration; command_configuration.read_fixed_number_of_rows = true; command_configuration.number_of_rows_to_read = rows_to_read; - Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process), log, std::move(tasks), command_configuration, process_pool)); + Pipe pipe(std::make_unique(context, configuration.format, sample_block, std::move(process), std::move(tasks), command_configuration, process_pool)); if (configuration.implicit_key) pipe.addTransform(std::make_shared(block, pipe.getHeader())); @@ -190,6 +197,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) .command_termination_timeout = config.getUInt64(settings_config_prefix + ".command_termination_timeout", 10), .max_command_execution_time = max_command_execution_time, .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false), + .send_chunk_header = config.getBool(settings_config_prefix + ".send_chunk_header", false) }; return std::make_unique(dict_struct, configuration, sample_block, context); diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.h b/src/Dictionaries/ExecutablePoolDictionarySource.h index b80122fb56f..2ccb4ce4b8d 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.h +++ b/src/Dictionaries/ExecutablePoolDictionarySource.h @@ -27,14 +27,16 @@ class ExecutablePoolDictionarySource final : public IDictionarySource public: struct Configuration { - const String command; - const String format; - const size_t pool_size; - const size_t command_termination_timeout; - const size_t max_command_execution_time; + String command; + String format; + size_t pool_size; + size_t command_termination_timeout; + size_t max_command_execution_time; /// Implicit key means that the source script will return only values, /// and the correspondence to the requested keys is determined implicitly - by the order of rows in the result. - const bool implicit_key; + bool implicit_key; + /// Send number_of_rows\n before sending chunk to process + bool send_chunk_header; }; ExecutablePoolDictionarySource( diff --git a/src/Functions/array/arrayCompact.cpp b/src/Functions/array/arrayCompact.cpp index e0f73207da8..4126f403516 100644 --- a/src/Functions/array/arrayCompact.cpp +++ b/src/Functions/array/arrayCompact.cpp @@ -16,7 +16,6 @@ namespace ErrorCodes struct ArrayCompactImpl { - static bool useDefaultImplementationForConstants() { return true; } static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } @@ -129,6 +128,7 @@ struct ArrayCompactImpl { ColumnPtr res; + mapped = mapped->convertToFullColumnIfConst(); if (!(executeType< UInt8 >(mapped, array, res) || executeType< UInt16>(mapped, array, res) || executeType< UInt32>(mapped, array, res) || diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index 288422c1c9c..2d4928f35b6 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -19,7 +19,6 @@ namespace ErrorCodes */ struct ArrayCumSumNonNegativeImpl { - static bool useDefaultImplementationForConstants() { return true; } static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } @@ -100,6 +99,7 @@ struct ArrayCumSumNonNegativeImpl { ColumnPtr res; + mapped = mapped->convertToFullColumnIfConst(); if (executeType< UInt8 , UInt64>(mapped, array, res) || executeType< UInt16, UInt64>(mapped, array, res) || executeType< UInt32, UInt64>(mapped, array, res) || diff --git a/src/Functions/array/arrayDifference.cpp b/src/Functions/array/arrayDifference.cpp index 7d11c6e89c8..ce223b5c9d8 100644 --- a/src/Functions/array/arrayDifference.cpp +++ b/src/Functions/array/arrayDifference.cpp @@ -20,7 +20,6 @@ namespace ErrorCodes */ struct ArrayDifferenceImpl { - static bool useDefaultImplementationForConstants() { return true; } static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } @@ -129,6 +128,7 @@ struct ArrayDifferenceImpl { ColumnPtr res; + mapped = mapped->convertToFullColumnIfConst(); if (executeType< UInt8 , Int16>(mapped, array, res) || executeType< UInt16, Int32>(mapped, array, res) || executeType< UInt32, Int64>(mapped, array, res) || diff --git a/src/IO/ReadBufferFromString.h b/src/IO/ReadBufferFromString.h index fac1b230a2c..1f8319d9a33 100644 --- a/src/IO/ReadBufferFromString.h +++ b/src/IO/ReadBufferFromString.h @@ -2,18 +2,17 @@ #include - namespace DB { -/** Allows to read from std::string-like object. - */ +/// Allows to read from std::string-like object. class ReadBufferFromString : public ReadBufferFromMemory { public: /// std::string or something similar template ReadBufferFromString(const S & s) : ReadBufferFromMemory(s.data(), s.size()) {} -}; + explicit ReadBufferFromString(std::string_view s) : ReadBufferFromMemory(s.data(), s.size()) {} +}; } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index c26eb10e697..a2896127d04 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -786,7 +786,7 @@ void NO_INLINE Aggregator::executeWithoutKeyImpl( AggregatedDataWithoutKey & res, size_t rows, AggregateFunctionInstruction * aggregate_instructions, - Arena * arena) + Arena * arena) const { #if USE_EMBEDDED_COMPILER if constexpr (use_compiled_functions) @@ -865,7 +865,7 @@ void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl( void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns & aggregate_columns, Columns & materialized_columns, - AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) + AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) const { for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_columns[i].resize(params.aggregates[i].arguments.size()); @@ -917,7 +917,7 @@ void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & result, - ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) + ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) const { UInt64 num_rows = block.rows(); return executeOnBlock(block.getColumns(), num_rows, result, key_columns, aggregate_columns, no_more_keys); @@ -925,7 +925,7 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedDataVariants & result, - ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) + ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) const { /// `result` will destroy the states of aggregate functions in the destructor result.aggregator = this; @@ -1058,7 +1058,7 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData } -void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, const String & tmp_path) +void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, const String & tmp_path) const { Stopwatch watch; size_t rows = data_variants.size(); @@ -1130,7 +1130,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co } -void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) +void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants) const { String tmp_path = params.tmp_volume->getDisk()->getPath(); return writeToTemporaryFile(data_variants, tmp_path); @@ -1192,7 +1192,7 @@ template void Aggregator::writeToTemporaryFileImpl( AggregatedDataVariants & data_variants, Method & method, - IBlockOutputStream & out) + IBlockOutputStream & out) const { size_t max_temporary_block_size_rows = 0; size_t max_temporary_block_size_bytes = 0; @@ -2311,7 +2311,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( block.clear(); } -bool Aggregator::mergeBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) +bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) const { /// `result` will destroy the states of aggregate functions in the destructor result.aggregator = this; @@ -2661,7 +2661,7 @@ void NO_INLINE Aggregator::convertBlockToTwoLevelImpl( } -std::vector Aggregator::convertBlockToTwoLevel(const Block & block) +std::vector Aggregator::convertBlockToTwoLevel(const Block & block) const { if (!block) return {}; @@ -2753,7 +2753,7 @@ void Aggregator::destroyWithoutKey(AggregatedDataVariants & result) const } -void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) +void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) const { if (result.empty()) return; diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index fde6ba219df..e72fe4baea3 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -506,7 +506,7 @@ struct AggregatedDataVariants : private boost::noncopyable * But this can hardly be done simply because it is planned to put variable-length strings into the same pool. * In this case, the pool will not be able to know with what offsets objects are stored. */ - Aggregator * aggregator = nullptr; + const Aggregator * aggregator = nullptr; size_t keys_size{}; /// Number of keys. NOTE do we need this field? Sizes key_sizes; /// Dimensions of keys, if keys of fixed length @@ -975,11 +975,14 @@ public: /// Process one block. Return false if the processing should be aborted (with group_by_overflow_mode = 'break'). bool executeOnBlock(const Block & block, AggregatedDataVariants & result, ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, /// Passed to not create them anew for each block - bool & no_more_keys); + bool & no_more_keys) const; bool executeOnBlock(Columns columns, UInt64 num_rows, AggregatedDataVariants & result, ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, /// Passed to not create them anew for each block - bool & no_more_keys); + bool & no_more_keys) const; + + /// Used for aggregate projection. + bool mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) const; /** Convert the aggregation data structure into a block. * If overflow_row = true, then aggregates for rows that are not included in max_rows_to_group_by are put in the first block. @@ -996,8 +999,6 @@ public: /// Merge partially aggregated blocks separated to buckets into one data structure. void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads); - bool mergeBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys); - /// Merge several partially aggregated blocks into one. /// Precondition: for all blocks block.info.is_overflows flag must be the same. /// (either all blocks are from overflow data or none blocks are). @@ -1007,11 +1008,11 @@ public: /** Split block with partially-aggregated data to many blocks, as if two-level method of aggregation was used. * This is needed to simplify merging of that data with other results, that are already two-level. */ - std::vector convertBlockToTwoLevel(const Block & block); + std::vector convertBlockToTwoLevel(const Block & block) const; /// For external aggregation. - void writeToTemporaryFile(AggregatedDataVariants & data_variants, const String & tmp_path); - void writeToTemporaryFile(AggregatedDataVariants & data_variants); + void writeToTemporaryFile(AggregatedDataVariants & data_variants, const String & tmp_path) const; + void writeToTemporaryFile(AggregatedDataVariants & data_variants) const; bool hasTemporaryFiles() const { return !temporary_files.empty(); } @@ -1083,7 +1084,7 @@ private: Poco::Logger * log = &Poco::Logger::get("Aggregator"); /// For external aggregation. - TemporaryFiles temporary_files; + mutable TemporaryFiles temporary_files; #if USE_EMBEDDED_COMPILER std::shared_ptr compiled_aggregate_functions_holder; @@ -1106,7 +1107,7 @@ private: /** Call `destroy` methods for states of aggregate functions. * Used in the exception handler for aggregation, since RAII in this case is not applicable. */ - void destroyAllAggregateStates(AggregatedDataVariants & result); + void destroyAllAggregateStates(AggregatedDataVariants & result) const; /// Process one data block, aggregate the data into a hash table. @@ -1136,7 +1137,7 @@ private: AggregatedDataWithoutKey & res, size_t rows, AggregateFunctionInstruction * aggregate_instructions, - Arena * arena); + Arena * arena) const; static void executeOnIntervalWithoutKeyImpl( AggregatedDataWithoutKey & res, @@ -1149,7 +1150,7 @@ private: void writeToTemporaryFileImpl( AggregatedDataVariants & data_variants, Method & method, - IBlockOutputStream & out); + IBlockOutputStream & out) const; /// Merge NULL key data from hash table `src` into `dst`. template @@ -1304,7 +1305,7 @@ private: AggregateColumns & aggregate_columns, Columns & materialized_columns, AggregateFunctionInstructions & instructions, - NestedColumnsHolder & nested_columns_holder); + NestedColumnsHolder & nested_columns_holder) const; void addSingleKeyToAggregateColumns( const AggregatedDataVariants & data_variants, diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index fd02aa4abec..c1e4bb86895 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -1105,7 +1105,8 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti } catch (...) { - tryLogCurrentException(__PRETTY_FUNCTION__); + if (errno != ENODATA) /// Ok for thermal sensors. + tryLogCurrentException(__PRETTY_FUNCTION__); /// Files maybe re-created on module load/unload try @@ -1144,7 +1145,8 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti } catch (...) { - tryLogCurrentException(__PRETTY_FUNCTION__); + if (errno != ENODATA) /// Ok for thermal sensors. + tryLogCurrentException(__PRETTY_FUNCTION__); /// Files can be re-created on: /// - module load/unload diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 27089ab8d37..9005b24b044 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -353,7 +353,8 @@ static bool isCompilableFunction(const ActionsDAG::Node & node, const std::unord static CompileDAG getCompilableDAG( const ActionsDAG::Node * root, - ActionsDAG::NodeRawConstPtrs & children) + ActionsDAG::NodeRawConstPtrs & children, + const std::unordered_set & lazy_executed_nodes) { /// Extract CompileDAG from root actions dag node. @@ -376,29 +377,29 @@ static CompileDAG getCompilableDAG( const auto * node = frame.node; bool is_compilable_constant = isCompilableConstant(*node); - bool is_compilable_function = isCompilableFunction(*node, {}); + bool is_compilable_function = isCompilableFunction(*node, lazy_executed_nodes); if (!is_compilable_function || is_compilable_constant) { - CompileDAG::Node compile_node; - compile_node.function = node->function_base; - compile_node.result_type = node->result_type; + CompileDAG::Node compile_node; + compile_node.function = node->function_base; + compile_node.result_type = node->result_type; - if (is_compilable_constant) - { - compile_node.type = CompileDAG::CompileType::CONSTANT; - compile_node.column = node->column; - } - else - { + if (is_compilable_constant) + { + compile_node.type = CompileDAG::CompileType::CONSTANT; + compile_node.column = node->column; + } + else + { compile_node.type = CompileDAG::CompileType::INPUT; children.emplace_back(node); - } + } - visited_node_to_compile_dag_position[node] = dag.getNodesCount(); - dag.addNode(std::move(compile_node)); - stack.pop(); - continue; + visited_node_to_compile_dag_position[node] = dag.getNodesCount(); + dag.addNode(std::move(compile_node)); + stack.pop(); + continue; } while (frame.next_child_to_visit < node->children.size()) @@ -568,7 +569,7 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression, const for (auto & node : nodes_to_compile) { NodeRawConstPtrs new_children; - auto dag = getCompilableDAG(node, new_children); + auto dag = getCompilableDAG(node, new_children, lazy_executed_nodes); if (dag.getInputNodesCount() == 0) continue; diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index eb7824a1124..09d0ea3a4b7 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1198,8 +1198,8 @@ class ExternalLoader::PeriodicUpdater : private boost::noncopyable public: static constexpr UInt64 check_period_sec = 5; - PeriodicUpdater(LoadablesConfigReader & config_files_reader_, LoadingDispatcher & loading_dispatcher_) - : config_files_reader(config_files_reader_), loading_dispatcher(loading_dispatcher_) + PeriodicUpdater(LoadablesConfigReader & config_files_reader_, LoadingDispatcher & loading_dispatcher_, std::recursive_mutex & config_mutex_) + : config_files_reader(config_files_reader_), loading_dispatcher(loading_dispatcher_), config_mutex(config_mutex_) { } @@ -1242,8 +1242,11 @@ private: while (!event.wait_for(lock, std::chrono::seconds(check_period_sec), pred)) { lock.unlock(); - loading_dispatcher.setConfiguration(config_files_reader.read()); - loading_dispatcher.reloadOutdated(); + { + std::lock_guard config_lock{config_mutex}; + loading_dispatcher.setConfiguration(config_files_reader.read()); + loading_dispatcher.reloadOutdated(); + } lock.lock(); } } @@ -1251,6 +1254,7 @@ private: LoadablesConfigReader & config_files_reader; LoadingDispatcher & loading_dispatcher; + std::recursive_mutex & config_mutex; mutable std::mutex mutex; bool enabled = false; ThreadFromGlobalPool thread; @@ -1264,7 +1268,7 @@ ExternalLoader::ExternalLoader(const String & type_name_, Poco::Logger * log_) [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, type_name_, log_)) - , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) + , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher, config_mutex)) , type_name(type_name_) , log(log_) { @@ -1276,11 +1280,13 @@ scope_guard ExternalLoader::addConfigRepository(std::unique_ptrgetName(); + std::lock_guard lock{config_mutex}; config_files_reader->addConfigRepository(std::move(repository)); reloadConfig(name); return [this, ptr, name]() { + std::lock_guard config_lock{config_mutex}; config_files_reader->removeConfigRepository(ptr); reloadConfig(name); }; @@ -1379,7 +1385,10 @@ ReturnType ExternalLoader::load(const FilterByNameFunction & filter) const template ReturnType ExternalLoader::loadOrReload(const String & name) const { - loading_dispatcher->setConfiguration(config_files_reader->read()); + { + std::lock_guard lock{config_mutex}; + loading_dispatcher->setConfiguration(config_files_reader->read()); + } auto result = loading_dispatcher->tryLoadOrReload(name, WAIT); checkLoaded(result, true); return convertTo(result); @@ -1388,7 +1397,10 @@ ReturnType ExternalLoader::loadOrReload(const String & name) const template ReturnType ExternalLoader::loadOrReload(const FilterByNameFunction & filter) const { - loading_dispatcher->setConfiguration(config_files_reader->read()); + { + std::lock_guard lock{config_mutex}; + loading_dispatcher->setConfiguration(config_files_reader->read()); + } auto results = loading_dispatcher->tryLoadOrReload(filter, WAIT); checkLoaded(results, true); return convertTo(results); @@ -1476,16 +1488,19 @@ void ExternalLoader::checkLoaded(const ExternalLoader::LoadResults & results, void ExternalLoader::reloadConfig() const { + std::lock_guard lock{config_mutex}; loading_dispatcher->setConfiguration(config_files_reader->read()); } void ExternalLoader::reloadConfig(const String & repository_name) const { + std::lock_guard lock{config_mutex}; loading_dispatcher->setConfiguration(config_files_reader->read(repository_name)); } void ExternalLoader::reloadConfig(const String & repository_name, const String & path) const { + std::lock_guard lock{config_mutex}; loading_dispatcher->setConfiguration(config_files_reader->read(repository_name, path)); } diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index 2ea79d3aade..1fa11807eba 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -219,6 +219,14 @@ private: LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; + /// We have to read configuration from LoadablesConfigReader and load configuration using LoadingDispatcher atomically. + /// Otherwise we can read configuration in one thread, then read and load newer configuration in another thread, + /// and then load outdated configuration from the first thread. + /// Remarkably, each class (LoadablesConfigReader, LoadingDispatcher, PeriodicUpdater) has own mutex for own purposes, + /// but it does not save from complex logical race conditions. + /// TODO Refactor dictionaries loading and get rid of this. + mutable std::recursive_mutex config_mutex; + class LoadablesConfigReader; std::unique_ptr config_files_reader; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index bcf6792b799..aab851955b6 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -338,7 +338,7 @@ BlockIO InterpreterSystemQuery::execute() { #if defined(__ELF__) && !defined(__FreeBSD__) getContext()->checkAccess(AccessType::SYSTEM_RELOAD_SYMBOLS); - (void)SymbolIndex::instance(true); + SymbolIndex::reload(); break; #else throw Exception("SYSTEM RELOAD SYMBOLS is not supported on current platform", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 7802bf6e3bf..a8a93e53663 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -395,9 +395,14 @@ AggregatingTransform::AggregatingTransform(Block header, AggregatingTransformPar } AggregatingTransform::AggregatingTransform( - Block header, AggregatingTransformParamsPtr params_, ManyAggregatedDataPtr many_data_, - size_t current_variant, size_t max_threads_, size_t temporary_data_merge_threads_) - : IProcessor({std::move(header)}, {params_->getHeader()}), params(std::move(params_)) + Block header, + AggregatingTransformParamsPtr params_, + ManyAggregatedDataPtr many_data_, + size_t current_variant, + size_t max_threads_, + size_t temporary_data_merge_threads_) + : IProcessor({std::move(header)}, {params_->getHeader()}) + , params(std::move(params_)) , key_columns(params->params.keys_size) , aggregate_columns(params->params.aggregates_size) , many_data(std::move(many_data_)) @@ -525,7 +530,7 @@ void AggregatingTransform::consume(Chunk chunk) { auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); block = materializeBlock(block); - if (!params->aggregator.mergeBlock(block, variants, no_more_keys)) + if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys)) is_consume_finished = true; } else @@ -547,7 +552,7 @@ void AggregatingTransform::initGenerate() if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) { if (params->only_merge) - params->aggregator.mergeBlock(getInputs().front().getHeader(), variants, no_more_keys); + params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys); else params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); } diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 9512a7a2811..1639bc4df4b 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -27,15 +27,38 @@ public: class IBlockInputStream; using BlockInputStreamPtr = std::shared_ptr; +using AggregatorList = std::list; +using AggregatorListPtr = std::shared_ptr; + struct AggregatingTransformParams { Aggregator::Params params; - Aggregator aggregator; + + /// Each params holds a list of aggregators which are used in query. It's needed because we need + /// to use a pointer of aggregator to proper destroy complex aggregation states on exception + /// (See comments in AggregatedDataVariants). However, this pointer might not be valid because + /// we can have two different aggregators at the same time due to mixed pipeline of aggregate + /// projections, and one of them might gets destroyed before used. + AggregatorListPtr aggregator_list_ptr; + Aggregator & aggregator; bool final; bool only_merge = false; AggregatingTransformParams(const Aggregator::Params & params_, bool final_) - : params(params_), aggregator(params), final(final_) {} + : params(params_) + , aggregator_list_ptr(std::make_shared()) + , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), params)) + , final(final_) + { + } + + AggregatingTransformParams(const Aggregator::Params & params_, const AggregatorListPtr & aggregator_list_ptr_, bool final_) + : params(params_) + , aggregator_list_ptr(aggregator_list_ptr_) + , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), params)) + , final(final_) + { + } Block getHeader() const { return aggregator.getHeader(final); } @@ -82,9 +105,13 @@ public: AggregatingTransform(Block header, AggregatingTransformParamsPtr params_); /// For Parallel aggregating. - AggregatingTransform(Block header, AggregatingTransformParamsPtr params_, - ManyAggregatedDataPtr many_data, size_t current_variant, - size_t max_threads, size_t temporary_data_merge_threads); + AggregatingTransform( + Block header, + AggregatingTransformParamsPtr params_, + ManyAggregatedDataPtr many_data, + size_t current_variant, + size_t max_threads, + size_t temporary_data_merge_threads); ~AggregatingTransform() override; String getName() const override { return "AggregatingTransform"; } diff --git a/src/Storages/ExecutablePoolSettings.cpp b/src/Storages/ExecutableSettings.cpp similarity index 79% rename from src/Storages/ExecutablePoolSettings.cpp rename to src/Storages/ExecutableSettings.cpp index 8951c8edabf..136357eb6f8 100644 --- a/src/Storages/ExecutablePoolSettings.cpp +++ b/src/Storages/ExecutableSettings.cpp @@ -1,4 +1,4 @@ -#include "ExecutablePoolSettings.h" +#include "ExecutableSettings.h" #include @@ -14,9 +14,9 @@ namespace ErrorCodes extern const int UNKNOWN_SETTING; } -IMPLEMENT_SETTINGS_TRAITS(ExecutablePoolSettingsTraits, LIST_OF_EXECUTABLE_POOL_SETTINGS); +IMPLEMENT_SETTINGS_TRAITS(ExecutableSettingsTraits, LIST_OF_EXECUTABLE_SETTINGS); -void ExecutablePoolSettings::loadFromQuery(ASTStorage & storage_def) +void ExecutableSettings::loadFromQuery(ASTStorage & storage_def) { if (storage_def.settings) { diff --git a/src/Storages/ExecutablePoolSettings.h b/src/Storages/ExecutableSettings.h similarity index 62% rename from src/Storages/ExecutablePoolSettings.h rename to src/Storages/ExecutableSettings.h index 6de9b0f0e6c..9c0cfc05fa5 100644 --- a/src/Storages/ExecutablePoolSettings.h +++ b/src/Storages/ExecutableSettings.h @@ -8,15 +8,16 @@ namespace DB class ASTStorage; -#define LIST_OF_EXECUTABLE_POOL_SETTINGS(M) \ +#define LIST_OF_EXECUTABLE_SETTINGS(M) \ + M(UInt64, send_chunk_header, false, "Send number_of_rows\n before sending chunk to process", 0) \ M(UInt64, pool_size, 16, "Processes pool size. If size == 0, then there is no size restrictions", 0) \ M(UInt64, max_command_execution_time, 10, "Max command execution time in seconds.", 0) \ M(UInt64, command_termination_timeout, 10, "Command termination timeout in seconds.", 0) \ -DECLARE_SETTINGS_TRAITS(ExecutablePoolSettingsTraits, LIST_OF_EXECUTABLE_POOL_SETTINGS) +DECLARE_SETTINGS_TRAITS(ExecutableSettingsTraits, LIST_OF_EXECUTABLE_SETTINGS) /// Settings for ExecutablePool engine. -struct ExecutablePoolSettings : public BaseSettings +struct ExecutableSettings : public BaseSettings { void loadFromQuery(ASTStorage & storage_def); }; diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index d659259e1a9..d0d845ed6a3 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -135,10 +135,11 @@ void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_e String offsets_name = Nested::extractTableName(name); auto offset_it = offset_columns.find(offsets_name); - if (offset_it != offset_columns.end()) + const auto * array_type = typeid_cast(type.get()); + if (offset_it != offset_columns.end() && array_type) { + const auto & nested_type = array_type->getNestedType(); ColumnPtr offsets_column = offset_it->second; - DataTypePtr nested_type = typeid_cast(*type).getNestedType(); size_t nested_rows = typeid_cast(*offsets_column).getData().back(); ColumnPtr nested_column = diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ff469078630..602a5f50be4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -908,6 +908,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { /// Check extra parts at different disks, in order to not allow to miss data parts at undefined disks. std::unordered_set defined_disk_names; + for (const auto & disk_ptr : disks) defined_disk_names.insert(disk_ptr->getName()); @@ -917,9 +918,10 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { for (const auto it = disk->iterateDirectory(relative_data_path); it->isValid(); it->next()) { - MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(it->name(), &part_info, format_version)) - throw Exception("Part " + backQuote(it->name()) + " was found on disk " + backQuote(disk_name) + " which is not defined in the storage policy", ErrorCodes::UNKNOWN_DISK); + if (MergeTreePartInfo::tryParsePartName(it->name(), format_version)) + throw Exception(ErrorCodes::UNKNOWN_DISK, + "Part {} was found on disk {} which is not defined in the storage policy", + backQuote(it->name()), backQuote(disk_name)); } } } @@ -930,10 +932,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (auto disk_it = disks.rbegin(); disk_it != disks.rend(); ++disk_it) { auto disk_ptr = *disk_it; + for (auto it = disk_ptr->iterateDirectory(relative_data_path); it->isValid(); it->next()) { /// Skip temporary directories, file 'format_version.txt' and directory 'detached'. - if (startsWith(it->name(), "tmp") || it->name() == MergeTreeData::FORMAT_VERSION_FILE_NAME || it->name() == MergeTreeData::DETACHED_DIR_NAME) + if (startsWith(it->name(), "tmp") + || it->name() == MergeTreeData::FORMAT_VERSION_FILE_NAME + || it->name() == MergeTreeData::DETACHED_DIR_NAME) continue; if (!startsWith(it->name(), MergeTreeWriteAheadLog::WAL_FILE_NAME)) @@ -980,25 +985,34 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { pool.scheduleOrThrowOnError([&] { - const auto & part_name = part_names_with_disk.first; - const auto part_disk_ptr = part_names_with_disk.second; + const auto & [part_name, part_disk_ptr] = part_names_with_disk; - MergeTreePartInfo part_info; - if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) + auto part_opt = MergeTreePartInfo::tryParsePartName(part_name, format_version); + + if (!part_opt) return; auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); - auto part = createPart(part_name, part_info, single_disk_volume, part_name); + auto part = createPart(part_name, *part_opt, single_disk_volume, part_name); bool broken = false; String part_path = fs::path(relative_data_path) / part_name; String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; + if (part_disk_ptr->exists(marker_path)) { - LOG_WARNING(log, "Detaching stale part {}{}, which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_WARNING(log, + "Detaching stale part {}{}, which should have been deleted after a move. That can only happen " + "after unclean restart of ClickHouse after move of a part having an operation blocking that " + "stale copy of part.", + getFullPathOnDisk(part_disk_ptr), part_name); + std::lock_guard loading_lock(mutex); + broken_parts_to_detach.push_back(part); + ++suspicious_broken_parts; + return; } @@ -1026,25 +1040,34 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Ignore broken parts that can appear as a result of hard server restart. if (broken) { - LOG_ERROR(log, "Detaching broken part {}{}. If it happened after update, it is likely because of backward incompability. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); + LOG_ERROR(log, + "Detaching broken part {}{}. If it happened after update, it is likely because of backward " + "incompatibility. You need to resolve this manually", + getFullPathOnDisk(part_disk_ptr), part_name); + std::lock_guard loading_lock(mutex); + broken_parts_to_detach.push_back(part); + ++suspicious_broken_parts; return; } + if (!part->index_granularity_info.is_adaptive) has_non_adaptive_parts.store(true, std::memory_order_relaxed); else has_adaptive_parts.store(true, std::memory_order_relaxed); part->modification_time = part_disk_ptr->getLastModified(fs::path(relative_data_path) / part_name).epochTime(); + /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->setState(DataPartState::Committed); std::lock_guard loading_lock(mutex); + if (!data_parts_indexes.insert(part).second) - throw Exception("Part " + part->name + " already exists", ErrorCodes::DUPLICATE_DATA_PART); + throw Exception(ErrorCodes::DUPLICATE_DATA_PART, "Part {} already exists", part->name); addPartContributionToDataVolume(part); }); @@ -3328,11 +3351,12 @@ RestoreDataTasks MergeTreeData::restoreDataPartsFromBackup(const BackupPtr & bac Strings part_names = backup->list(data_path_in_backup); for (const String & part_name : part_names) { - MergeTreePartInfo part_info; - if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) + const auto part_info = MergeTreePartInfo::tryParsePartName(part_name, format_version); + + if (!part_info) continue; - if (!partition_ids.empty() && !partition_ids.contains(part_info.partition_id)) + if (!partition_ids.empty() && !partition_ids.contains(part_info->partition_id)) continue; UInt64 total_size_of_part = 0; @@ -3369,7 +3393,7 @@ RestoreDataTasks MergeTreeData::restoreDataPartsFromBackup(const BackupPtr & bac } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto part = createPart(part_name, part_info, single_disk_volume, relative_temp_part_dir); + auto part = createPart(part_name, *part_info, single_disk_volume, relative_temp_part_dir); part->loadColumnsChecksumsIndexes(false, true); renameTempPartAndAdd(part, increment); }; @@ -3581,11 +3605,10 @@ std::vector MergeTreeData::getDetachedParts() const { for (auto it = disk->iterateDirectory(detached_path); it->isValid(); it->next()) { - res.emplace_back(); - auto & part = res.back(); - - DetachedPartInfo::tryParseDetachedPartName(it->name(), part, format_version); + auto part = DetachedPartInfo::parseDetachedPartName(it->name(), format_version); part.disk = disk->getName(); + + res.push_back(std::move(part)); } } } @@ -3656,7 +3679,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const validateDetachedPartName(part_id); renamed_parts.addPart(part_id, "attaching_" + part_id); - if (MergeTreePartInfo::tryParsePartName(part_id, nullptr, format_version)) + if (MergeTreePartInfo::tryParsePartName(part_id, format_version)) name_to_disk[part_id] = getDiskForPart(part_id, source_dir); } else @@ -3672,12 +3695,11 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const for (auto it = disk->iterateDirectory(relative_data_path + source_dir); it->isValid(); it->next()) { const String & name = it->name(); - MergeTreePartInfo part_info; // TODO what if name contains "_tryN" suffix? /// Parts with prefix in name (e.g. attaching_1_3_3_0, deleting_1_3_3_0) will be ignored - if (!MergeTreePartInfo::tryParsePartName(name, &part_info, format_version) - || part_info.partition_id != partition_id) + if (auto part_opt = MergeTreePartInfo::tryParsePartName(name, format_version); + !part_opt || part_opt->partition_id != partition_id) { continue; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 92529b00faa..b1a8ee5b986 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -267,6 +267,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( auto many_data = std::make_shared(projection_pipe.numOutputPorts() + ordinary_pipe.numOutputPorts()); size_t counter = 0; + AggregatorListPtr aggregator_list_ptr = std::make_shared(); + // TODO apply in_order_optimization here auto build_aggregate_pipe = [&](Pipe & pipe, bool projection) { @@ -306,7 +308,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( settings.min_count_to_compile_aggregate_expression, header_before_aggregation); // The source header is also an intermediate header - transform_params = std::make_shared(std::move(params), query_info.projection->aggregate_final); + transform_params = std::make_shared( + std::move(params), aggregator_list_ptr, query_info.projection->aggregate_final); /// This part is hacky. /// We want AggregatingTransform to work with aggregate states instead of normal columns. @@ -336,7 +339,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); - transform_params = std::make_shared(std::move(params), query_info.projection->aggregate_final); + transform_params = std::make_shared( + std::move(params), aggregator_list_ptr, query_info.projection->aggregate_final); } pipe.resize(pipe.numOutputPorts(), true, true); diff --git a/src/Storages/MergeTree/MergeTreePartInfo.cpp b/src/Storages/MergeTree/MergeTreePartInfo.cpp index 6a98e666c34..7e140ec2375 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -15,13 +15,12 @@ namespace ErrorCodes MergeTreePartInfo MergeTreePartInfo::fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version) { - MergeTreePartInfo part_info; - if (!tryParsePartName(part_name, &part_info, format_version)) - throw Exception("Unexpected part name: " + part_name, ErrorCodes::BAD_DATA_PART_NAME); - return part_info; + if (auto part_opt = tryParsePartName(part_name, format_version)) + return *part_opt; + else + throw Exception(ErrorCodes::BAD_DATA_PART_NAME, "Unexpected part name: {}", part_name); } - void MergeTreePartInfo::validatePartitionID(const String & partition_id, MergeTreeDataFormatVersion format_version) { if (partition_id.empty()) @@ -43,22 +42,26 @@ void MergeTreePartInfo::validatePartitionID(const String & partition_id, MergeTr } -bool MergeTreePartInfo::tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version) +std::optional MergeTreePartInfo::tryParsePartName( + std::string_view part_name, MergeTreeDataFormatVersion format_version) { ReadBufferFromString in(part_name); String partition_id; + if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { UInt32 min_yyyymmdd = 0; UInt32 max_yyyymmdd = 0; + if (!tryReadIntText(min_yyyymmdd, in) || !checkChar('_', in) || !tryReadIntText(max_yyyymmdd, in) || !checkChar('_', in)) { - return false; + return std::nullopt; } + partition_id = toString(min_yyyymmdd / 100); } else @@ -76,9 +79,7 @@ bool MergeTreePartInfo::tryParsePartName(const String & part_name, MergeTreePart /// Sanity check if (partition_id.empty()) - { - return false; - } + return std::nullopt; Int64 min_block_num = 0; Int64 max_block_num = 0; @@ -91,14 +92,12 @@ bool MergeTreePartInfo::tryParsePartName(const String & part_name, MergeTreePart || !checkChar('_', in) || !tryReadIntText(level, in)) { - return false; + return std::nullopt; } /// Sanity check if (min_block_num > max_block_num) - { - return false; - } + return std::nullopt; if (!in.eof()) { @@ -106,29 +105,30 @@ bool MergeTreePartInfo::tryParsePartName(const String & part_name, MergeTreePart || !tryReadIntText(mutation, in) || !in.eof()) { - return false; + return std::nullopt; } } - if (part_info) + MergeTreePartInfo part_info; + + part_info.partition_id = std::move(partition_id); + part_info.min_block = min_block_num; + part_info.max_block = max_block_num; + + if (level == LEGACY_MAX_LEVEL) { - part_info->partition_id = std::move(partition_id); - part_info->min_block = min_block_num; - part_info->max_block = max_block_num; - if (level == LEGACY_MAX_LEVEL) - { - /// We (accidentally) had two different max levels until 21.6 and it might cause logical errors like - /// "Part 20170601_20170630_0_2_999999999 intersects 201706_0_1_4294967295". - /// So we replace unexpected max level to make contains(...) method and comparison operators work - /// correctly with such virtual parts. On part name serialization we will use legacy max level to keep the name unchanged. - part_info->use_leagcy_max_level = true; - level = MAX_LEVEL; - } - part_info->level = level; - part_info->mutation = mutation; + /// We (accidentally) had two different max levels until 21.6 and it might cause logical errors like + /// "Part 20170601_20170630_0_2_999999999 intersects 201706_0_1_4294967295". + /// So we replace unexpected max level to make contains(...) method and comparison operators work + /// correctly with such virtual parts. On part name serialization we will use legacy max level to keep the name unchanged. + part_info.use_leagcy_max_level = true; + level = MAX_LEVEL; } - return true; + part_info.level = level; + part_info.mutation = mutation; + + return part_info; } @@ -235,55 +235,75 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con return wb.str(); } - -const std::vector DetachedPartInfo::DETACH_REASONS = - { - "broken", - "unexpected", - "noquorum", - "ignored", - "broken-on-start", - "clone", - "attaching", - "deleting", - "tmp-fetch", - }; - -bool DetachedPartInfo::tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, - MergeTreeDataFormatVersion format_version) +DetachedPartInfo DetachedPartInfo::parseDetachedPartName( + std::string_view dir_name, MergeTreeDataFormatVersion format_version) { + DetachedPartInfo part_info; + part_info.dir_name = dir_name; - /// First, try to find known prefix and parse dir_name as _. + /// First, try to find known prefix and parse dir_name as _. /// Arbitrary strings are not allowed for partition_id, so known_prefix cannot be confused with partition_id. - for (const auto & known_prefix : DETACH_REASONS) + for (std::string_view known_prefix : DETACH_REASONS) { - if (dir_name.starts_with(known_prefix) && known_prefix.size() < dir_name.size() && dir_name[known_prefix.size()] == '_') + if (dir_name.starts_with(known_prefix) + && known_prefix.size() < dir_name.size() + && dir_name[known_prefix.size()] == '_') { part_info.prefix = known_prefix; - String part_name = dir_name.substr(known_prefix.size() + 1); - bool parsed = MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version); - return part_info.valid_name = parsed; + + const std::string_view part_name = dir_name.substr(known_prefix.size() + 1); + + if (auto part_opt = MergeTreePartInfo::tryParsePartName(part_name, format_version)) + { + part_info.valid_name = true; + part_info.addParsedPartInfo(*part_opt); + } + else + part_info.valid_name = false; + + return part_info; } } /// Next, try to parse dir_name as . - if (MergeTreePartInfo::tryParsePartName(dir_name, &part_info, format_version)) - return part_info.valid_name = true; + if (auto part_opt = MergeTreePartInfo::tryParsePartName(dir_name, format_version)) + { + part_info.valid_name = true; + part_info.addParsedPartInfo(*part_opt); + return part_info; + } /// Next, as _. Use entire name as prefix if it fails. part_info.prefix = dir_name; - const auto first_separator = dir_name.find_first_of('_'); + + const size_t first_separator = dir_name.find_first_of('_'); + if (first_separator == String::npos) - return part_info.valid_name = false; + { + part_info.valid_name = false; + return part_info; + } - const auto part_name = dir_name.substr(first_separator + 1, - dir_name.size() - first_separator - 1); - if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) - return part_info.valid_name = false; + const std::string_view part_name = dir_name.substr( + first_separator + 1, + dir_name.size() - first_separator - 1); - part_info.prefix = dir_name.substr(0, first_separator); - return part_info.valid_name = true; + if (auto part_opt = MergeTreePartInfo::tryParsePartName(part_name, format_version)) + { + part_info.valid_name = true; + part_info.prefix = dir_name.substr(0, first_separator); + part_info.addParsedPartInfo(*part_opt); + } + else + part_info.valid_name = false; + + return part_info; } +void DetachedPartInfo::addParsedPartInfo(const MergeTreePartInfo& part) +{ + // Both class are aggregates so it's ok. + static_cast(*this) = part; +} } diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 181fef7990c..73761048035 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -1,8 +1,10 @@ #pragma once #include +#include #include #include +#include #include #include #include @@ -98,7 +100,8 @@ struct MergeTreePartInfo static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version); // -V1071 - static bool tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version); + static std::optional tryParsePartName( + std::string_view part_name, MergeTreeDataFormatVersion format_version); static void parseMinMaxDatesFromPartName(const String & part_name, DayNum & min_date, DayNum & max_date); @@ -122,11 +125,27 @@ struct DetachedPartInfo : public MergeTreePartInfo /// If false, MergeTreePartInfo is in invalid state (directory name was not successfully parsed). bool valid_name; - static const std::vector DETACH_REASONS; + static constexpr auto DETACH_REASONS = std::to_array({ + "broken", + "unexpected", + "noquorum", + "ignored", + "broken-on-start", + "clone", + "attaching", + "deleting", + "tmp-fetch" + }); /// NOTE: It may parse part info incorrectly. - /// For example, if prefix contain '_' or if DETACH_REASONS doesn't contain prefix. - static bool tryParseDetachedPartName(const String & dir_name, DetachedPartInfo & part_info, MergeTreeDataFormatVersion format_version); + /// For example, if prefix contains '_' or if DETACH_REASONS doesn't contain prefix. + // This method has different semantics with MergeTreePartInfo::tryParsePartName. + // Detached parts are always parsed regardless of their validity. + // DetachedPartInfo::valid_name field specifies whether parsing was successful or not. + static DetachedPartInfo parseDetachedPartName(std::string_view dir_name, MergeTreeDataFormatVersion format_version); + +private: + void addParsedPartInfo(const MergeTreePartInfo& part); }; using DetachedPartsInfo = std::vector; diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index c8bbb703999..c8bc215dd6c 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -193,10 +193,6 @@ void StorageDictionary::startup() void StorageDictionary::removeDictionaryConfigurationFromRepository() { - if (remove_repository_callback_executed) - return; - - remove_repository_callback_executed = true; remove_repository_callback.reset(); } diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index d074dec2c34..c7693bdae9a 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -73,7 +73,6 @@ private: Poco::Timestamp update_time; LoadablesConfigurationPtr configuration; - std::atomic remove_repository_callback_executed = false; scope_guard remove_repository_callback; void removeDictionaryConfigurationFromRepository(); diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index 4b0aaf6caea..56b410a3ea4 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -56,7 +56,7 @@ StorageExecutable::StorageExecutable( const std::vector & arguments_, const String & format_, const std::vector & input_queries_, - const ExecutablePoolSettings & pool_settings_, + const ExecutableSettings & settings_, const ColumnsDescription & columns, const ConstraintsDescription & constraints) : IStorage(table_id_) @@ -64,9 +64,9 @@ StorageExecutable::StorageExecutable( , arguments(arguments_) , format(format_) , input_queries(input_queries_) - , pool_settings(pool_settings_) + , settings(settings_) /// If pool size == 0 then there is no size restrictions. Poco max size of semaphore is integer type. - , process_pool(std::make_shared(pool_settings.pool_size == 0 ? std::numeric_limits::max() : pool_settings.pool_size)) + , process_pool(std::make_shared(settings.pool_size == 0 ? std::numeric_limits::max() : settings.pool_size)) , log(&Poco::Logger::get("StorageExecutablePool")) { StorageInMemoryMetadata storage_metadata; @@ -114,15 +114,15 @@ Pipe StorageExecutable::read( { bool result = process_pool->tryBorrowObject(process, [&config, this]() { - config.terminate_in_destructor_strategy = ShellCommand::DestructorStrategy{ true /*terminate_in_destructor*/, pool_settings.command_termination_timeout }; + config.terminate_in_destructor_strategy = ShellCommand::DestructorStrategy{ true /*terminate_in_destructor*/, settings.command_termination_timeout }; auto shell_command = ShellCommand::execute(config); return shell_command; - }, pool_settings.max_command_execution_time * 10000); + }, settings.max_command_execution_time * 1000); if (!result) throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Could not get process from pool, max command execution timeout exceeded {} seconds", - pool_settings.max_command_execution_time); + settings.max_command_execution_time); } else { @@ -137,6 +137,8 @@ Pipe StorageExecutable::read( BlockInputStreamPtr input_stream = inputs[i]; WriteBufferFromFile * write_buffer = nullptr; + bool send_chunk_header = settings.send_chunk_header; + if (i == 0) { write_buffer = &process->in; @@ -151,14 +153,22 @@ Pipe StorageExecutable::read( write_buffer = &it->second; } - ShellCommandSource::SendDataTask task = [input_stream, write_buffer, context, is_executable_pool, this]() + ShellCommandSource::SendDataTask task = [input_stream, write_buffer, context, is_executable_pool, send_chunk_header, this]() { auto output_stream = context->getOutputStream(format, *write_buffer, input_stream->getHeader().cloneEmpty()); input_stream->readPrefix(); output_stream->writePrefix(); while (auto block = input_stream->read()) + { + if (send_chunk_header) + { + writeText(block.rows(), *write_buffer); + writeChar('\n', *write_buffer); + } + output_stream->write(block); + } input_stream->readSuffix(); output_stream->writeSuffix(); @@ -183,7 +193,7 @@ Pipe StorageExecutable::read( configuration.read_number_of_rows_from_process_output = true; } - Pipe pipe(std::make_unique(context, format, std::move(sample_block), std::move(process), log, std::move(tasks), configuration, process_pool)); + Pipe pipe(std::make_unique(context, format, std::move(sample_block), std::move(process), std::move(tasks), configuration, process_pool)); return pipe; } @@ -232,7 +242,7 @@ void registerStorageExecutable(StorageFactory & factory) if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; - ExecutablePoolSettings pool_settings; + ExecutableSettings pool_settings; pool_settings.max_command_execution_time = max_command_execution_time; if (args.storage_def->settings) pool_settings.loadFromQuery(*args.storage_def); diff --git a/src/Storages/StorageExecutable.h b/src/Storages/StorageExecutable.h index dd986ee3956..c67891d876f 100644 --- a/src/Storages/StorageExecutable.h +++ b/src/Storages/StorageExecutable.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace DB @@ -55,7 +55,7 @@ protected: const std::vector & arguments_, const String & format_, const std::vector & input_queries_, - const ExecutablePoolSettings & pool_settings_, + const ExecutableSettings & settings_, const ColumnsDescription & columns, const ConstraintsDescription & constraints); @@ -64,7 +64,7 @@ private: std::vector arguments; String format; std::vector input_queries; - ExecutablePoolSettings pool_settings; + ExecutableSettings settings; std::shared_ptr process_pool; Poco::Logger * log; }; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9ac4864b87f..7e10e9ee62f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1161,10 +1161,8 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) const UInt64 parts_to_fetch_blocks = std::accumulate(parts_to_fetch.cbegin(), parts_to_fetch.cend(), 0, [&](UInt64 acc, const String& part_name) { - MergeTreePartInfo part_info; - - if (MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) - return acc + part_info.getBlocksCount(); + if (const auto part_info = MergeTreePartInfo::tryParsePartName(part_name, format_version)) + return acc + part_info->getBlocksCount(); LOG_ERROR(log, "Unexpected part name: {}", part_name); return acc; @@ -1454,13 +1452,12 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo for (const DiskPtr & disk : getStoragePolicy()->getDisks()) for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / "detached/"); it->isValid(); it->next()) { - MergeTreePartInfo part_info; + const auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version); - if (!MergeTreePartInfo::tryParsePartName(it->name(), &part_info, format_version) || - part_info.partition_id != actual_part_info.partition_id) + if (!part_info || part_info->partition_id != actual_part_info.partition_id) continue; - const String part_old_name = part_info.getPartName(); + const String part_old_name = part_info->getPartName(); const String part_path = fs::path("detached") / part_old_name; const VolumePtr volume = std::make_shared("volume_" + part_old_name, disk); @@ -6310,6 +6307,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( String partition_prefix = partition_id + "_"; zkutil::AsyncResponses get_futures; + for (const String & block_id : blocks) { if (startsWith(block_id, partition_prefix)) @@ -6328,9 +6326,10 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( continue; ReadBufferFromString buf(result.data); - MergeTreePartInfo part_info; - bool parsed = MergeTreePartInfo::tryParsePartName(result.data, &part_info, format_version); - if (!parsed || (min_block_num <= part_info.min_block && part_info.max_block <= max_block_num)) + + const auto part_info = MergeTreePartInfo::tryParsePartName(result.data, format_version); + + if (!part_info || (min_block_num <= part_info->min_block && part_info->max_block <= max_block_num)) ops.emplace_back(zkutil::makeRemoveRequest(path, -1)); } } @@ -7422,12 +7421,15 @@ bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_n bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & partition_name) { fs::directory_iterator dir_end; + for (const std::string & path : getDataPaths()) { for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it) { - MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(dir_it->path().filename(), &part_info, format_version) && part_info.partition_id == partition_name) + const String file_name = dir_it->path().filename().string(); + auto part_info = MergeTreePartInfo::tryParsePartName(file_name, format_version); + + if (part_info && part_info->partition_id == partition_name) return true; } } diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index ba2ed4c55cd..a278b9d3dec 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f3a41ba6a25..c3136c4fd52 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1162,6 +1162,9 @@ def main(args): total_tests_run += do_run_tests( jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_tests, sequential_tests, args.parallel) + if server_died.is_set(): + exit_code.value = 1 + if args.hung_check: # Some queries may execute in background for some time after test was finished. This is normal. diff --git a/tests/integration/README.md b/tests/integration/README.md index ed96eafdef8..1b1e7f3e052 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -66,7 +66,7 @@ For tests that use common docker compose files you may need to set up their path ### Running with runner script The only requirement is fresh configured docker and -docker pull yandex/clickhouse-integration-tests-runner +docker pull clickhouse/integration-tests-runner Notes: * If you want to run integration tests without `sudo` you have to add your user to docker group `sudo usermod -aG docker $USER`. [More information](https://docs.docker.com/install/linux/linux-postinstall/) about docker configuration. @@ -122,7 +122,7 @@ You can just open shell inside a container by overwritting the command: The main container used for integration tests lives in `docker/test/integration/Dockerfile`. Rebuild it with ``` cd docker/test/integration -docker build -t yandex/clickhouse-integration-test . +docker build -t clickhouse/integration-test . ``` The helper container used by the `runner` script is in `docker/test/integration/runner/Dockerfile`. diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index bf7549a83c4..f17eb84e5f3 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -207,11 +207,11 @@ class ClickhouseIntegrationTestsRunner: @staticmethod def get_images_names(): - return ["yandex/clickhouse-integration-tests-runner", "yandex/clickhouse-mysql-golang-client", - "yandex/clickhouse-mysql-java-client", "yandex/clickhouse-mysql-js-client", - "yandex/clickhouse-mysql-php-client", "yandex/clickhouse-postgresql-java-client", - "yandex/clickhouse-integration-test", "yandex/clickhouse-kerberos-kdc", - "yandex/clickhouse-integration-helper", ] + return ["clickhouse/integration-tests-runner", "clickhouse/mysql-golang-client", + "clickhouse/mysql-java-client", "clickhouse/mysql-js-client", + "clickhouse/mysql-php-client", "clickhouse/postgresql-java-client", + "clickhouse/integration-test", "clickhouse/kerberos-kdc", + "clickhouse/integration-helper", ] def _can_run_with(self, path, opt): @@ -343,7 +343,7 @@ class ClickhouseIntegrationTestsRunner: image_cmd = '' if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-image-version'): for img in self.get_images_names(): - if img == "yandex/clickhouse-integration-tests-runner": + if img == "clickhouse/integration-tests-runner": runner_version = self.get_single_image_version() logging.info("Can run with custom docker image version %s", runner_version) image_cmd += ' --docker-image-version={} '.format(runner_version) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 165e2175cf2..d1240999274 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -532,7 +532,7 @@ class ClickHouseCluster: binary_path = binary_path[:-len('-server')] env_variables['keeper_binary'] = binary_path - env_variables['image'] = "yandex/clickhouse-integration-test:" + self.docker_base_tag + env_variables['image'] = "clickhouse/integration-test:" + self.docker_base_tag env_variables['user'] = str(os.getuid()) env_variables['keeper_fs'] = 'bind' for i in range(1, 4): @@ -757,7 +757,7 @@ class ClickHouseCluster: with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, with_hdfs=False, with_kerberized_hdfs=False, with_mongo=False, with_mongo_secure=False, with_nginx=False, with_redis=False, with_minio=False, with_cassandra=False, with_jdbc_bridge=False, - hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, + hostname=None, env_variables=None, image="clickhouse/integration-test", tag=None, stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, zookeeper_docker_compose_path=None, minio_certs_dir=None, use_keeper=True, main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True): @@ -1801,7 +1801,7 @@ class ClickHouseInstance: clickhouse_start_command=CLICKHOUSE_START_COMMAND, main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True, hostname=None, env_variables=None, - image="yandex/clickhouse-integration-test", tag="latest", + image="clickhouse/integration-test", tag="latest", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): self.name = name diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 040d49b992f..af0942c9b94 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -195,21 +195,21 @@ class _NetworkManager: print("Error removing network blocade container, will try again", str(ex)) time.sleep(i) - image = subprocess.check_output("docker images -q yandex/clickhouse-integration-helper 2>/dev/null", shell=True) + image = subprocess.check_output("docker images -q clickhouse/integration-helper 2>/dev/null", shell=True) if not image.strip(): print("No network image helper, will try download") # for some reason docker api may hang if image doesn't exist, so we download it # before running for i in range(5): try: - subprocess.check_call("docker pull yandex/clickhouse-integration-helper", shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call("docker pull clickhouse/integration-helper", shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL break except: time.sleep(i) else: - raise Exception("Cannot pull yandex/clickhouse-integration-helper image") + raise Exception("Cannot pull clickhouse/integration-helper image") - self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper', + self._container = self._docker_client.containers.run('clickhouse/integration-helper', auto_remove=True, command=('sleep %s' % self.container_exit_timeout), detach=True, network_mode='host') diff --git a/tests/integration/runner b/tests/integration/runner index 2143d7ebf29..3ff982a9913 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -19,7 +19,7 @@ CONFIG_DIR_IN_REPO = "programs/server" INTERGATION_DIR_IN_REPO = "tests/integration" SRC_DIR_IN_REPO = "src" -DIND_INTEGRATION_TESTS_IMAGE_NAME = "yandex/clickhouse-integration-tests-runner" +DIND_INTEGRATION_TESTS_IMAGE_NAME = "clickhouse/integration-tests-runner" def check_args_and_update_paths(args): if args.clickhouse_root: @@ -225,19 +225,19 @@ if __name__ == "__main__": if args.docker_compose_images_tags is not None: for img_tag in args.docker_compose_images_tags: [image, tag] = img_tag.split(":") - if image == "yandex/clickhouse-mysql-golang-client": + if image == "clickhouse/mysql-golang-client": env_tags += "-e {}={} ".format("DOCKER_MYSQL_GOLANG_CLIENT_TAG", tag) - elif image == "yandex/clickhouse-mysql-java-client": + elif image == "clickhouse/mysql-java-client": env_tags += "-e {}={} ".format("DOCKER_MYSQL_JAVA_CLIENT_TAG", tag) - elif image == "yandex/clickhouse-mysql-js-client": + elif image == "clickhouse/mysql-js-client": env_tags += "-e {}={} ".format("DOCKER_MYSQL_JS_CLIENT_TAG", tag) - elif image == "yandex/clickhouse-mysql-php-client": + elif image == "clickhouse/mysql-php-client": env_tags += "-e {}={} ".format("DOCKER_MYSQL_PHP_CLIENT_TAG", tag) - elif image == "yandex/clickhouse-postgresql-java-client": + elif image == "clickhouse/postgresql-java-client": env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag) - elif image == "yandex/clickhouse-integration-test": + elif image == "clickhouse/integration-test": env_tags += "-e {}={} ".format("DOCKER_BASE_TAG", tag) - elif image == "yandex/clickhouse-kerberos-kdc": + elif image == "clickhouse/kerberos-kdc": env_tags += "-e {}={}".format("DOCKER_KERBEROS_KDC_TAG", tag) else: logging.info("Unknown image %s" % (image)) diff --git a/tests/integration/test_mysql_protocol/golang.reference b/tests/integration/test_mysql_protocol/golang.reference index db16a5a6925..4069b2a086a 100644 --- a/tests/integration/test_mysql_protocol/golang.reference +++ b/tests/integration/test_mysql_protocol/golang.reference @@ -1,7 +1,7 @@ Columns: a Column types: -a BIGINT +a UNSIGNED BIGINT Result: 0 1 @@ -10,7 +10,7 @@ name a Column types: name CHAR -a TINYINT +a UNSIGNED TINYINT Result: tables 1 tables 1 @@ -19,6 +19,6 @@ a b Column types: a CHAR -b TINYINT +b UNSIGNED TINYINT Result: тест 1 diff --git a/tests/queries/0_stateless/00336_shard_stack_trace.reference b/tests/queries/0_stateless/00336_shard_stack_trace.reference index ca4011b20db..a956634eefc 100644 --- a/tests/queries/0_stateless/00336_shard_stack_trace.reference +++ b/tests/queries/0_stateless/00336_shard_stack_trace.reference @@ -2,6 +2,6 @@ 1 Ok 1 -2 +3 Ok -2 +3 diff --git a/tests/queries/0_stateless/00602_throw_if.sh b/tests/queries/0_stateless/00602_throw_if.sh index fe8feab0303..1e8850028c4 100755 --- a/tests/queries/0_stateless/00602_throw_if.sh +++ b/tests/queries/0_stateless/00602_throw_if.sh @@ -8,5 +8,5 @@ default_exception_message="Value passed to 'throwIf' function is non zero" custom_exception_message="Number equals 1000000" ${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --query="SELECT throwIf(number = 1000000) FROM system.numbers" 2>&1 | grep -cF "$default_exception_message" -${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --query="SELECT throwIf(number = 1000000, '$custom_exception_message') FROM system.numbers" 2>&1 | grep -cF "$custom_exception_message" +${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --query="SELECT throwIf(number = 1000000, '$custom_exception_message') FROM system.numbers" 2>&1 | grep -v '^(query: ' | grep -cF "$custom_exception_message" ${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --query="SELECT sum(x = 0) FROM (SELECT throwIf(number = 1000000) AS x FROM numbers(1000000))" 2>&1 diff --git a/tests/queries/0_stateless/00763_lock_buffer.reference b/tests/queries/0_stateless/00763_lock_buffer_long.reference similarity index 100% rename from tests/queries/0_stateless/00763_lock_buffer.reference rename to tests/queries/0_stateless/00763_lock_buffer_long.reference diff --git a/tests/queries/0_stateless/00763_lock_buffer.sh b/tests/queries/0_stateless/00763_lock_buffer_long.sh similarity index 94% rename from tests/queries/0_stateless/00763_lock_buffer.sh rename to tests/queries/0_stateless/00763_lock_buffer_long.sh index 44660035208..363e3803983 100755 --- a/tests/queries/0_stateless/00763_lock_buffer.sh +++ b/tests/queries/0_stateless/00763_lock_buffer_long.sh @@ -18,7 +18,7 @@ function thread1() function thread2() { - seq 1 1000 | sed -r -e 's/.+/SELECT count() FROM buffer_00763_2;/' | ${CLICKHOUSE_CLIENT} --multiquery --server_logs_file='/dev/null' --ignore-error 2>&1 | grep -vP '^0$|^10$|^Received exception|^Code: 60|^Code: 218|^Code: 473' + seq 1 1000 | sed -r -e 's/.+/SELECT count() FROM buffer_00763_2;/' | ${CLICKHOUSE_CLIENT} --multiquery --server_logs_file='/dev/null' --ignore-error 2>&1 | grep -vP '^0$|^10$|^Received exception|^Code: 60|^Code: 218|^Code: 473' | grep -v '(query: ' } thread1 & diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh index 52a29c19be1..df2ac2cece8 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh @@ -13,4 +13,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) python3 "${CURDIR}"/00921_datetime64_compatibility_long.python \ | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ - | grep -v 'Received exception .*$' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' + | grep -v -e 'Received exception .*$' -e '^(query: ' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' diff --git a/tests/queries/0_stateless/00956_sensitive_data_masking.sh b/tests/queries/0_stateless/00956_sensitive_data_masking.sh index 7462dfd5585..e7179e1e002 100755 --- a/tests/queries/0_stateless/00956_sensitive_data_masking.sh +++ b/tests/queries/0_stateless/00956_sensitive_data_masking.sh @@ -37,7 +37,7 @@ echo 3 # failure at before query start $CLICKHOUSE_CLIENT \ --query="SELECT 'find_me_TOPSECRET=TOPSECRET' FROM non_existing_table FORMAT Null" \ - --log_queries=1 --ignore-error --multiquery >"$tmp_file" 2>&1 + --log_queries=1 --ignore-error --multiquery |& grep -v '^(query: ' > "$tmp_file" grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 3a' grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 3b' @@ -47,7 +47,7 @@ echo 4 # failure at the end of query $CLICKHOUSE_CLIENT \ --query="SELECT 'find_me_TOPSECRET=TOPSECRET', intDiv( 100, number - 10) FROM numbers(11) FORMAT Null" \ - --log_queries=1 --ignore-error --max_block_size=2 --multiquery >"$tmp_file" 2>&1 + --log_queries=1 --ignore-error --max_block_size=2 --multiquery |& grep -v '^(query: ' > "$tmp_file" grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 4a' grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 4b' @@ -57,7 +57,7 @@ echo 5 rm -f "$tmp_file2" >/dev/null 2>&1 bash -c "$CLICKHOUSE_CLIENT \ --query=\"select sleepEachRow(1) from numbers(10) where ignore('find_me_TOPSECRET=TOPSECRET')=0 and ignore('fwerkh_that_magic_string_make_me_unique') = 0 FORMAT Null\" \ - --log_queries=1 --ignore-error --multiquery >$tmp_file2 2>&1" & + --log_queries=1 --ignore-error --multiquery |& grep -v '^(query: ' > $tmp_file2" & rm -f "$tmp_file" >/dev/null 2>&1 # check that executing query doesn't expose secrets in processlist diff --git a/tests/queries/0_stateless/01020_function_array_compact.reference b/tests/queries/0_stateless/01020_function_array_compact.reference index 6627a437251..4a6265b4f55 100644 --- a/tests/queries/0_stateless/01020_function_array_compact.reference +++ b/tests/queries/0_stateless/01020_function_array_compact.reference @@ -7,3 +7,5 @@ [1,2,1] [2,1] [1,2,3,4,5] +[0] +[0] diff --git a/tests/queries/0_stateless/01020_function_array_compact.sql b/tests/queries/0_stateless/01020_function_array_compact.sql index eea69dcb6da..d4aaa4d3fca 100644 --- a/tests/queries/0_stateless/01020_function_array_compact.sql +++ b/tests/queries/0_stateless/01020_function_array_compact.sql @@ -7,3 +7,5 @@ select arrayCompact([1,1,2]); select arrayCompact([1,2,1]); select arrayCompact([2,1,1]); select arrayCompact([1,2,2,3,3,3,4,4,4,4,5,5,5,5,5]); +SELECT arrayCompact(x->0, [NULL]); +SELECT toString(arrayCompact(x->0, [NULL])); diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh index 6355f790e23..27c9ba55858 100755 --- a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -14,20 +14,20 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t2 (x UInt64, s Array(Nullab function thread_detach_attach { while true; do - $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (219)' + $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: (219)' -e '(query: ' sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (82)' + $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: (219)' -e '(query: ' sleep 0.0$RANDOM done } function thread_rename { while true; do - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' sleep 0.0$RANDOM done } @@ -42,4 +42,4 @@ sleep 1 $CLICKHOUSE_CLIENT --query "DETACH DATABASE IF EXISTS test_01150" $CLICKHOUSE_CLIENT --query "ATTACH DATABASE IF NOT EXISTS test_01150" -$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01150"; +$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01150" diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.reference b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.reference index 29f6f801044..11c8996c021 100644 --- a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.reference +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.reference @@ -1,19 +1,24 @@ none Received exception from server: Code: 57. Error: Received from localhost:9000. Error: There was an error on [localhost:9000]: Code: 57. Error: Table default.none already exists. (TABLE_ALREADY_EXISTS) +(query: create table none on cluster test_shard_localhost (n int) engine=Memory;) Received exception from server: Code: 159. Error: Received from localhost:9000. Error: Watching task is executing longer than distributed_ddl_task_timeout (=1) seconds. There are 1 unfinished hosts (0 of them are currently active), they are going to execute the query in background. (TIMEOUT_EXCEEDED) +(query: drop table if exists none on cluster test_unavailable_shard;) throw localhost 9000 0 0 0 Received exception from server: Code: 57. Error: Received from localhost:9000. Error: There was an error on [localhost:9000]: Code: 57. Error: Table default.throw already exists. (TABLE_ALREADY_EXISTS) +(query: create table throw on cluster test_shard_localhost (n int) engine=Memory format Null;) localhost 9000 0 1 0 Received exception from server: Code: 159. Error: Received from localhost:9000. Error: Watching task is executing longer than distributed_ddl_task_timeout (=1) seconds. There are 1 unfinished hosts (0 of them are currently active), they are going to execute the query in background. (TIMEOUT_EXCEEDED) +(query: drop table if exists throw on cluster test_unavailable_shard;) null_status_on_timeout localhost 9000 0 0 0 Received exception from server: Code: 57. Error: Received from localhost:9000. Error: There was an error on [localhost:9000]: Code: 57. Error: Table default.null_status already exists. (TABLE_ALREADY_EXISTS) +(query: create table null_status on cluster test_shard_localhost (n int) engine=Memory format Null;) localhost 9000 0 1 0 localhost 1 \N \N 1 0 never_throw diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh index 483979d00db..319b9928e75 100755 --- a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh @@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +TMP_OUT=$(mktemp "$CURDIR/01175_distributed_ddl_output_mode_long.XXXXXX") +trap 'rm -f ${TMP_OUT:?}' EXIT # We execute a distributed DDL query with timeout 1 to check that one host is unavailable and will time out and other complete successfully. # But sometimes one second is not enough even for healthy host to succeed. Repeat the test in this case. @@ -16,11 +18,11 @@ function run_until_out_contains() for _ in {1..20} do - "$@" > "${CLICKHOUSE_TMP}/out" 2>&1 - if grep -q "$PATTERN" "${CLICKHOUSE_TMP}/out" + "$@" > "$TMP_OUT" 2>&1 + if grep -q "$PATTERN" "$TMP_OUT" then - cat "${CLICKHOUSE_TMP}/out" - break; + cat "$TMP_OUT" + break fi done } diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 6248813c9ba..b7cc7a3bc32 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -11,7 +11,7 @@ function thread() while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | - grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time' + grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|^\(query: ' done } diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh index 667a612ff23..8bcb5f8a7e8 100755 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -25,28 +25,28 @@ function thread_drop { function thread_rename { while true; do - $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' sleep 0.0$RANDOM done } function thread_select { while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' sleep 0.0$RANDOM done } function thread_insert { while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: '| grep -v -P 'Code: (60|218)' sleep 0.0$RANDOM done } function thread_insert_select { while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' sleep 0.0$RANDOM done } diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh index 9f31fcb6da1..441399a500e 100755 --- a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh @@ -18,35 +18,35 @@ function thread_create { function thread_drop { while true; do - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' sleep 0.0$RANDOM done } function thread_rename { while true; do - $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' sleep 0.0$RANDOM done } function thread_select { while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' sleep 0.0$RANDOM done } function thread_insert { while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' sleep 0.0$RANDOM done } function thread_insert_select { while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' sleep 0.0$RANDOM done } diff --git a/tests/queries/0_stateless/01529_bad_memory_tracking.sh b/tests/queries/0_stateless/01529_bad_memory_tracking.sh index 5ad2535074a..6f614d8329f 100755 --- a/tests/queries/0_stateless/01529_bad_memory_tracking.sh +++ b/tests/queries/0_stateless/01529_bad_memory_tracking.sh @@ -7,5 +7,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh for _ in {1..10}; do - ${CLICKHOUSE_CLIENT} --max_memory_usage '10G' --query "SELECT i FROM generateRandom('i Array(Int8)', 1, 1, 1048577) LIMIT 65536" 2>&1 | grep -v -P '^(Received exception from server|Code: 241)' ||: + ${CLICKHOUSE_CLIENT} --max_memory_usage '10G' --query "SELECT i FROM generateRandom('i Array(Int8)', 1, 1, 1048577) LIMIT 65536" |& grep -v -e 'Received exception from server' -e 'Code: 241' -e '(query: ' done +exit 0 diff --git a/tests/queries/0_stateless/01710_join_use_nulls.reference b/tests/queries/0_stateless/01707_join_use_nulls.reference similarity index 100% rename from tests/queries/0_stateless/01710_join_use_nulls.reference rename to tests/queries/0_stateless/01707_join_use_nulls.reference diff --git a/tests/queries/0_stateless/01710_join_use_nulls.sql b/tests/queries/0_stateless/01707_join_use_nulls.sql similarity index 100% rename from tests/queries/0_stateless/01710_join_use_nulls.sql rename to tests/queries/0_stateless/01707_join_use_nulls.sql diff --git a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.reference b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql new file mode 100644 index 00000000000..734aa659146 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql @@ -0,0 +1,9 @@ +drop table if exists t; + +create table t (x UInt32) engine = MergeTree order by tuple() settings index_granularity = 8; +insert into t select number from numbers(100); +alter table t add projection p (select uniqHLL12(x)); +insert into t select number + 100 from numbers(100); +select uniqHLL12(x) from t settings allow_experimental_projection_optimization = 1, max_bytes_to_read=400, max_block_size=8; -- { serverError 307; } + +drop table if exists t; diff --git a/tests/queries/0_stateless/01822_short_circuit.reference b/tests/queries/0_stateless/01822_short_circuit.reference index 96c4e161244..949d2fa4985 100644 --- a/tests/queries/0_stateless/01822_short_circuit.reference +++ b/tests/queries/0_stateless/01822_short_circuit.reference @@ -1803,3 +1803,33 @@ Decimal32 \N \N \N +0 +1 +0 +1 +0 +0 +1 +1 +1 +1 +1 +1 +3 +3 +5 +5 +7 +7 +9 +9 +1 +1 +3 +3 +5 +5 +7 +7 +9 +9 diff --git a/tests/queries/0_stateless/01822_short_circuit.sql b/tests/queries/0_stateless/01822_short_circuit.sql index fe8a0315d4a..1f0e04cb4b5 100644 --- a/tests/queries/0_stateless/01822_short_circuit.sql +++ b/tests/queries/0_stateless/01822_short_circuit.sql @@ -148,3 +148,8 @@ select if(isNull(x), Null, 42 / x) from (select CAST(materialize(Null), 'Nullabl select if(isNull(x), Null, x / 0) from (select CAST(materialize(Null), 'Nullable(Decimal32(2))') as x); select if(isNull(x), Null, intDiv(42, x)) from (select CAST(materialize(Null), 'Nullable(Int64)') as x); + +select number % 2 and toLowCardinality(number) from numbers(5); +select number % 2 or toLowCardinality(number) from numbers(5); +select if(toLowCardinality(number) % 2, number, number + 1) from numbers(10); +select multiIf(toLowCardinality(number) % 2, number, number + 1) from numbers(10); diff --git a/tests/queries/0_stateless/02001_select_with_filter.reference b/tests/queries/0_stateless/02001_select_with_filter.reference index 9d104af5e8c..92af7d1d38f 100644 --- a/tests/queries/0_stateless/02001_select_with_filter.reference +++ b/tests/queries/0_stateless/02001_select_with_filter.reference @@ -1,2 +1,4 @@ 98 2450 +\N +2500 diff --git a/tests/queries/0_stateless/02001_select_with_filter.sql b/tests/queries/0_stateless/02001_select_with_filter.sql index 4d10f86ed96..70152db83f8 100644 --- a/tests/queries/0_stateless/02001_select_with_filter.sql +++ b/tests/queries/0_stateless/02001_select_with_filter.sql @@ -1,3 +1,4 @@ SELECT argMax(number, number + 1) FILTER(WHERE number != 99) FROM numbers(100) ; SELECT sum(number) FILTER(WHERE number % 2 == 0) FROM numbers(100); -SELECT sumIfOrNull(number, number % 2 == 1) FILTER(WHERE number % 2 == 0) FROM numbers(100); -- { serverError 184 } +SELECT sumIfOrNull(number, number % 2 == 1) FILTER(WHERE 0) FROM numbers(100); +SELECT sumIfOrNull(number, number % 2 == 1) FILTER(WHERE 1) FROM numbers(100); diff --git a/tests/queries/0_stateless/02017_columns_with_dot_2.reference b/tests/queries/0_stateless/02017_columns_with_dot_2.reference new file mode 100644 index 00000000000..8d43601632c --- /dev/null +++ b/tests/queries/0_stateless/02017_columns_with_dot_2.reference @@ -0,0 +1,2 @@ +123 asd [1,2] +123 asd [1,2] 0 diff --git a/tests/queries/0_stateless/02017_columns_with_dot_2.sql b/tests/queries/0_stateless/02017_columns_with_dot_2.sql new file mode 100644 index 00000000000..eefe52b74f3 --- /dev/null +++ b/tests/queries/0_stateless/02017_columns_with_dot_2.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS test_nested; + +CREATE TABLE test_nested +( + `id` String, + `with_dot.str` String, + `with_dot.array` Array(Int32) +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_nested VALUES('123', 'asd', [1,2]); +SELECT * FROM test_nested; + +ALTER TABLE test_nested ADD COLUMN `with_dot.bool` UInt8; +SELECT * FROM test_nested; + +DROP TABLE test_nested; diff --git a/tests/queries/0_stateless/02024_compile_expressions_with_short_circuit_evaluation.reference b/tests/queries/0_stateless/02024_compile_expressions_with_short_circuit_evaluation.reference new file mode 100644 index 00000000000..af23232bb2e --- /dev/null +++ b/tests/queries/0_stateless/02024_compile_expressions_with_short_circuit_evaluation.reference @@ -0,0 +1,3 @@ +-- { echo } +select 1+number+multiIf(number == 1, cityHash64(number), number) from numbers(1) settings compile_expressions=1, min_count_to_compile_expression=0; +1 diff --git a/tests/queries/0_stateless/02024_compile_expressions_with_short_circuit_evaluation.sql b/tests/queries/0_stateless/02024_compile_expressions_with_short_circuit_evaluation.sql new file mode 100644 index 00000000000..113d0d9d4f7 --- /dev/null +++ b/tests/queries/0_stateless/02024_compile_expressions_with_short_circuit_evaluation.sql @@ -0,0 +1,2 @@ +-- { echo } +select 1+number+multiIf(number == 1, cityHash64(number), number) from numbers(1) settings compile_expressions=1, min_count_to_compile_expression=0; diff --git a/tests/queries/0_stateless/02025_nested_func_for_if_combinator.reference b/tests/queries/0_stateless/02025_nested_func_for_if_combinator.reference new file mode 100644 index 00000000000..2b3b5b30715 --- /dev/null +++ b/tests/queries/0_stateless/02025_nested_func_for_if_combinator.reference @@ -0,0 +1,12 @@ +-- { echo } +SELECT uniqCombinedIfMerge(n) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +5 +SELECT uniqCombinedIfMergeIf(n, last > 50) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +0 +SELECT uniqCombinedIfMergeIf(n, last > 50) FILTER(WHERE last>50) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); -- { serverError ILLEGAL_AGGREGATION } +SELECT uniqCombinedIfMerge(n) FILTER(WHERE last>50) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +0 +SELECT uniqCombinedIfMergeIf(n, last > 5) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +5 +SELECT uniqCombinedIfMergeIfIf(n, last > 5) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); -- { serverError ILLEGAL_AGGREGATION } +SELECT uniqCombinedIfMergeIfIf(n, last > 5, 1) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); -- { serverError ILLEGAL_AGGREGATION } diff --git a/tests/queries/0_stateless/02025_nested_func_for_if_combinator.sql b/tests/queries/0_stateless/02025_nested_func_for_if_combinator.sql new file mode 100644 index 00000000000..4811023c198 --- /dev/null +++ b/tests/queries/0_stateless/02025_nested_func_for_if_combinator.sql @@ -0,0 +1,8 @@ +-- { echo } +SELECT uniqCombinedIfMerge(n) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +SELECT uniqCombinedIfMergeIf(n, last > 50) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +SELECT uniqCombinedIfMergeIf(n, last > 50) FILTER(WHERE last>50) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); -- { serverError ILLEGAL_AGGREGATION } +SELECT uniqCombinedIfMerge(n) FILTER(WHERE last>50) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +SELECT uniqCombinedIfMergeIf(n, last > 5) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); +SELECT uniqCombinedIfMergeIfIf(n, last > 5) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); -- { serverError ILLEGAL_AGGREGATION } +SELECT uniqCombinedIfMergeIfIf(n, last > 5, 1) FROM (SELECT uniqCombinedIfState(number, number % 2) AS n, max(number) AS last FROM numbers(10)); -- { serverError ILLEGAL_AGGREGATION } diff --git a/tests/queries/0_stateless/02026_arrayDifference_const.reference b/tests/queries/0_stateless/02026_arrayDifference_const.reference new file mode 100644 index 00000000000..7a0beed21a5 --- /dev/null +++ b/tests/queries/0_stateless/02026_arrayDifference_const.reference @@ -0,0 +1 @@ +[0,0] diff --git a/tests/queries/0_stateless/02026_arrayDifference_const.sql b/tests/queries/0_stateless/02026_arrayDifference_const.sql new file mode 100644 index 00000000000..55a48d2bedb --- /dev/null +++ b/tests/queries/0_stateless/02026_arrayDifference_const.sql @@ -0,0 +1 @@ +SELECT toString(arrayDifference(x->0, [1, 2])); diff --git a/tests/queries/0_stateless/02027_arrayCumSumNonNegative_const.reference b/tests/queries/0_stateless/02027_arrayCumSumNonNegative_const.reference new file mode 100644 index 00000000000..7a0beed21a5 --- /dev/null +++ b/tests/queries/0_stateless/02027_arrayCumSumNonNegative_const.reference @@ -0,0 +1 @@ +[0,0] diff --git a/tests/queries/0_stateless/02027_arrayCumSumNonNegative_const.sql b/tests/queries/0_stateless/02027_arrayCumSumNonNegative_const.sql new file mode 100644 index 00000000000..f9522073154 --- /dev/null +++ b/tests/queries/0_stateless/02027_arrayCumSumNonNegative_const.sql @@ -0,0 +1 @@ +SELECT toString(arrayCumSumNonNegative(x->0, [1, 2])); diff --git a/tests/testflows/aes_encryption/aes_encryption_env/clickhouse-service.yml b/tests/testflows/aes_encryption/aes_encryption_env/clickhouse-service.yml index 0789decf022..0c9352dbc0b 100644 --- a/tests/testflows/aes_encryption/aes_encryption_env/clickhouse-service.yml +++ b/tests/testflows/aes_encryption/aes_encryption_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/datetime64_extended_range/datetime64_extended_range_env/clickhouse-service.yml b/tests/testflows/datetime64_extended_range/datetime64_extended_range_env/clickhouse-service.yml index 0789decf022..0c9352dbc0b 100644 --- a/tests/testflows/datetime64_extended_range/datetime64_extended_range_env/clickhouse-service.yml +++ b/tests/testflows/datetime64_extended_range/datetime64_extended_range_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/example/example_env/clickhouse-service.yml b/tests/testflows/example/example_env/clickhouse-service.yml index 0789decf022..0c9352dbc0b 100644 --- a/tests/testflows/example/example_env/clickhouse-service.yml +++ b/tests/testflows/example/example_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/extended_precision_data_types/extended-precision-data-type_env/clickhouse-service.yml b/tests/testflows/extended_precision_data_types/extended-precision-data-type_env/clickhouse-service.yml index fdd4a8057a9..afb31f77c94 100644 --- a/tests/testflows/extended_precision_data_types/extended-precision-data-type_env/clickhouse-service.yml +++ b/tests/testflows/extended_precision_data_types/extended-precision-data-type_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/kerberos/kerberos_env/clickhouse-service.yml b/tests/testflows/kerberos/kerberos_env/clickhouse-service.yml index 14736a264b8..9f30ca3039a 100644 --- a/tests/testflows/kerberos/kerberos_env/clickhouse-service.yml +++ b/tests/testflows/kerberos/kerberos_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test:21454 + image: clickhouse/integration-test:21454 expose: - "9000" - "9009" diff --git a/tests/testflows/ldap/authentication/ldap_authentication_env/clickhouse-service.yml b/tests/testflows/ldap/authentication/ldap_authentication_env/clickhouse-service.yml index 0789decf022..0c9352dbc0b 100644 --- a/tests/testflows/ldap/authentication/ldap_authentication_env/clickhouse-service.yml +++ b/tests/testflows/ldap/authentication/ldap_authentication_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/ldap/external_user_directory/ldap_external_user_directory_env/clickhouse-service.yml b/tests/testflows/ldap/external_user_directory/ldap_external_user_directory_env/clickhouse-service.yml index 0789decf022..0c9352dbc0b 100644 --- a/tests/testflows/ldap/external_user_directory/ldap_external_user_directory_env/clickhouse-service.yml +++ b/tests/testflows/ldap/external_user_directory/ldap_external_user_directory_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/ldap/role_mapping/ldap_role_mapping_env/clickhouse-service.yml b/tests/testflows/ldap/role_mapping/ldap_role_mapping_env/clickhouse-service.yml index 0789decf022..0c9352dbc0b 100644 --- a/tests/testflows/ldap/role_mapping/ldap_role_mapping_env/clickhouse-service.yml +++ b/tests/testflows/ldap/role_mapping/ldap_role_mapping_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/map_type/map_type_env/clickhouse-service.yml b/tests/testflows/map_type/map_type_env/clickhouse-service.yml index fdd4a8057a9..afb31f77c94 100755 --- a/tests/testflows/map_type/map_type_env/clickhouse-service.yml +++ b/tests/testflows/map_type/map_type_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/rbac/rbac_env/clickhouse-service.yml b/tests/testflows/rbac/rbac_env/clickhouse-service.yml index 2d79443dcbb..ac52e3b83eb 100755 --- a/tests/testflows/rbac/rbac_env/clickhouse-service.yml +++ b/tests/testflows/rbac/rbac_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009" diff --git a/tests/testflows/runner b/tests/testflows/runner index 772a4d01a84..0208512762c 100755 --- a/tests/testflows/runner +++ b/tests/testflows/runner @@ -14,7 +14,7 @@ DEFAULT_CLICKHOUSE_ROOT = os.path.abspath(os.path.join(CUR_FILE_DIR, "../../")) CURRENT_WORK_DIR = os.getcwd() CONTAINER_NAME = "clickhouse_testflows_tests" -DIND_TESTFLOWS_TESTS_IMAGE_NAME = "yandex/clickhouse-testflows-runner" +DIND_TESTFLOWS_TESTS_IMAGE_NAME = "clickhouse/testflows-runner" def check_args_and_update_paths(args): if not os.path.isabs(args.binary): diff --git a/tests/testflows/window_functions/window_functions_env/clickhouse-service.yml b/tests/testflows/window_functions/window_functions_env/clickhouse-service.yml index fdd4a8057a9..afb31f77c94 100755 --- a/tests/testflows/window_functions/window_functions_env/clickhouse-service.yml +++ b/tests/testflows/window_functions/window_functions_env/clickhouse-service.yml @@ -2,7 +2,7 @@ version: '2.3' services: clickhouse: - image: yandex/clickhouse-integration-test + image: clickhouse/integration-test expose: - "9000" - "9009"