diff --git a/.github/actions/debug/action.yml b/.github/actions/debug/action.yml index e1fe3f28024..b45465809d2 100644 --- a/.github/actions/debug/action.yml +++ b/.github/actions/debug/action.yml @@ -4,15 +4,31 @@ description: Prints workflow debug info runs: using: "composite" steps: - - name: Print envs + - name: Envs, event.json and contexts shell: bash run: | - echo "::group::Envs" - env - echo "::endgroup::" - - name: Print Event.json - shell: bash - run: | - echo "::group::Event.json" + echo '::group::Environment variables' + env | sort + echo '::endgroup::' + + echo '::group::event.json' python3 -m json.tool "$GITHUB_EVENT_PATH" - echo "::endgroup::" + echo '::endgroup::' + + cat << 'EOF' + ::group::github context + ${{ toJSON(github) }} + ::endgroup:: + + ::group::env context + ${{ toJSON(env) }} + ::endgroup:: + + ::group::runner context + ${{ toJSON(runner) }} + ::endgroup:: + + ::group::job context + ${{ toJSON(job) }} + ::endgroup:: + EOF diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 23744dc7f8f..794aca4a515 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -27,6 +27,8 @@ jobs: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Labels check run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index 8d1e2055978..315673d4abc 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -33,6 +33,8 @@ jobs: clear-repository: true token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} fetch-depth: 0 + - name: Debug Info + uses: ./.github/actions/debug - name: Cherry pick run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 1fb6cb60e96..b6c460ab37c 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -56,13 +56,13 @@ jobs: GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} runs-on: [self-hosted, release-maker] steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} fetch-depth: 0 + - name: Debug Info + uses: ./.github/actions/debug - name: Prepare Release Info shell: bash run: | diff --git a/.github/workflows/docker_test_images.yml b/.github/workflows/docker_test_images.yml index 3fe1a8883c6..2138420f378 100644 --- a/.github/workflows/docker_test_images.yml +++ b/.github/workflows/docker_test_images.yml @@ -11,6 +11,7 @@ name: Build docker images required: false type: boolean default: false + jobs: DockerBuildAarch64: runs-on: [self-hosted, style-checker-aarch64] diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml index ecafde9e4cb..92e4ce10ade 100644 --- a/.github/workflows/jepsen.yml +++ b/.github/workflows/jepsen.yml @@ -8,27 +8,28 @@ on: # yamllint disable-line rule:truthy schedule: - cron: '0 */6 * * *' workflow_dispatch: + jobs: RunConfig: runs-on: [self-hosted, style-checker-aarch64] outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: PrepareRunConfig id: runconfig run: | echo "::group::configure CI run" python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow "$GITHUB_WORKFLOW" --outfile ${{ runner.temp }}/ci_run_data.json echo "::endgroup::" - + echo "::group::CI run configure results" python3 -m json.tool ${{ runner.temp }}/ci_run_data.json echo "::endgroup::" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2ce1124404f..b76bbbbbdbe 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -15,14 +15,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Merge sync PR run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml index 629cf79770e..45ce81c2caf 100644 --- a/.github/workflows/merge_queue.yml +++ b/.github/workflows/merge_queue.yml @@ -14,14 +14,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Cancel PR workflow run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 36fea39686f..1cea94e7500 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,14 +15,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: PrepareRunConfig id: runconfig run: | diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index dbc740ebc1b..acd392978b6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -25,14 +25,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Cancel previous Sync PR workflow run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ec119b6ff95..b884ebfe7a0 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -24,6 +24,8 @@ jobs: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Labels check run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/reusable_simple_job.yml b/.github/workflows/reusable_simple_job.yml index 4d48662ae4e..7df98d96f79 100644 --- a/.github/workflows/reusable_simple_job.yml +++ b/.github/workflows/reusable_simple_job.yml @@ -62,8 +62,6 @@ jobs: env: GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: @@ -72,6 +70,8 @@ jobs: submodules: ${{inputs.submodules}} fetch-depth: ${{inputs.checkout_depth}} filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Set build envs run: | cat >> "$GITHUB_ENV" << 'EOF' diff --git a/.gitmodules b/.gitmodules index 53ebde0cd3b..3aa2e4e8ea9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -170,9 +170,6 @@ [submodule "contrib/fast_float"] path = contrib/fast_float url = https://github.com/fastfloat/fast_float -[submodule "contrib/libpq"] - path = contrib/libpq - url = https://github.com/ClickHouse/libpq [submodule "contrib/NuRaft"] path = contrib/NuRaft url = https://github.com/ClickHouse/NuRaft @@ -369,3 +366,6 @@ [submodule "contrib/numactl"] path = contrib/numactl url = https://github.com/ClickHouse/numactl.git +[submodule "contrib/postgres"] + path = contrib/postgres + url = https://github.com/ClickHouse/postgres.git diff --git a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h index acc79ec92b2..6e44d9f45b7 100644 --- a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h +++ b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h @@ -188,8 +188,9 @@ namespace Crypto pFile = fopen(keyFile.c_str(), "r"); if (pFile) { - pem_password_cb * pCB = pass.empty() ? (pem_password_cb *)0 : &passCB; - void * pPassword = pass.empty() ? (void *)0 : (void *)pass.c_str(); + pem_password_cb * pCB = &passCB; + static constexpr char * no_password = ""; + void * pPassword = pass.empty() ? (void *)no_password : (void *)pass.c_str(); if (readFunc(pFile, &pKey, pCB, pPassword)) { fclose(pFile); @@ -225,6 +226,13 @@ namespace Crypto error: if (pFile) fclose(pFile); + if (*ppKey) + { + if constexpr (std::is_same_v) + EVP_PKEY_free(*ppKey); + else + EC_KEY_free(*ppKey); + } throw OpenSSLException("EVPKey::loadKey(string)"); } @@ -286,6 +294,13 @@ namespace Crypto error: if (pBIO) BIO_free(pBIO); + if (*ppKey) + { + if constexpr (std::is_same_v) + EVP_PKEY_free(*ppKey); + else + EC_KEY_free(*ppKey); + } throw OpenSSLException("EVPKey::loadKey(stream)"); } diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h index c19eecf5c73..2c56875835e 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h @@ -248,6 +248,9 @@ namespace Net SSL_CTX * sslContext() const; /// Returns the underlying OpenSSL SSL Context object. + SSL_CTX * takeSslContext(); + /// Takes ownership of the underlying OpenSSL SSL Context object. + Usage usage() const; /// Returns whether the context is for use by a client or by a server /// and whether TLSv1 is required. @@ -401,6 +404,13 @@ namespace Net return _pSSLContext; } + inline SSL_CTX * Context::takeSslContext() + { + auto * result = _pSSLContext; + _pSSLContext = nullptr; + return result; + } + inline bool Context::extendedCertificateVerificationEnabled() const { diff --git a/base/poco/NetSSL_OpenSSL/src/Context.cpp b/base/poco/NetSSL_OpenSSL/src/Context.cpp index da1c121286b..69c88eef63a 100644 --- a/base/poco/NetSSL_OpenSSL/src/Context.cpp +++ b/base/poco/NetSSL_OpenSSL/src/Context.cpp @@ -106,6 +106,11 @@ Context::Context( Context::~Context() { + if (_pSSLContext == nullptr) + { + return; + } + try { SSL_CTX_free(_pSSLContext); diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d7489bc5c0e..c36ace61396 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -145,8 +145,13 @@ add_contrib (isa-l-cmake isa-l) add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arrow, libhdfs3 add_contrib (cppkafka-cmake cppkafka) -add_contrib (libpqxx-cmake libpqxx) -add_contrib (libpq-cmake libpq) + +option(ENABLE_LIBPQXX "Enable PostgreSQL" ${ENABLE_LIBRARIES}) +if (ENABLE_LIBPQXX) + add_contrib (postgres-cmake postgres) + add_contrib (libpqxx-cmake libpqxx) +endif() + add_contrib (rocksdb-cmake rocksdb) # requires: jemalloc, snappy, zlib, lz4, zstd, liburing add_contrib (nuraft-cmake NuRaft) add_contrib (fast_float-cmake fast_float) diff --git a/contrib/libpq b/contrib/libpq deleted file mode 160000 index 2446f2c8565..00000000000 --- a/contrib/libpq +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2446f2c85650b56df9d4ebc4c2ea7f4b01beee57 diff --git a/contrib/libpq-cmake/CMakeLists.txt b/contrib/libpq-cmake/CMakeLists.txt deleted file mode 100644 index 246e19593f6..00000000000 --- a/contrib/libpq-cmake/CMakeLists.txt +++ /dev/null @@ -1,78 +0,0 @@ -if (NOT ENABLE_LIBPQXX) - return() -endif() - -set(LIBPQ_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpq") - -set(SRCS - "${LIBPQ_SOURCE_DIR}/fe-auth.c" - "${LIBPQ_SOURCE_DIR}/fe-auth-scram.c" - "${LIBPQ_SOURCE_DIR}/fe-connect.c" - "${LIBPQ_SOURCE_DIR}/fe-exec.c" - "${LIBPQ_SOURCE_DIR}/fe-lobj.c" - "${LIBPQ_SOURCE_DIR}/fe-misc.c" - "${LIBPQ_SOURCE_DIR}/fe-print.c" - "${LIBPQ_SOURCE_DIR}/fe-trace.c" - "${LIBPQ_SOURCE_DIR}/fe-protocol3.c" - "${LIBPQ_SOURCE_DIR}/fe-secure.c" - "${LIBPQ_SOURCE_DIR}/fe-secure-common.c" - "${LIBPQ_SOURCE_DIR}/fe-secure-openssl.c" - "${LIBPQ_SOURCE_DIR}/legacy-pqsignal.c" - "${LIBPQ_SOURCE_DIR}/libpq-events.c" - "${LIBPQ_SOURCE_DIR}/pqexpbuffer.c" - - "${LIBPQ_SOURCE_DIR}/common/scram-common.c" - "${LIBPQ_SOURCE_DIR}/common/sha2.c" - "${LIBPQ_SOURCE_DIR}/common/sha1.c" - "${LIBPQ_SOURCE_DIR}/common/md5.c" - "${LIBPQ_SOURCE_DIR}/common/md5_common.c" - "${LIBPQ_SOURCE_DIR}/common/hmac_openssl.c" - "${LIBPQ_SOURCE_DIR}/common/cryptohash.c" - "${LIBPQ_SOURCE_DIR}/common/saslprep.c" - "${LIBPQ_SOURCE_DIR}/common/unicode_norm.c" - "${LIBPQ_SOURCE_DIR}/common/ip.c" - "${LIBPQ_SOURCE_DIR}/common/jsonapi.c" - "${LIBPQ_SOURCE_DIR}/common/wchar.c" - "${LIBPQ_SOURCE_DIR}/common/base64.c" - "${LIBPQ_SOURCE_DIR}/common/link-canary.c" - "${LIBPQ_SOURCE_DIR}/common/fe_memutils.c" - "${LIBPQ_SOURCE_DIR}/common/string.c" - "${LIBPQ_SOURCE_DIR}/common/pg_get_line.c" - "${LIBPQ_SOURCE_DIR}/common/stringinfo.c" - "${LIBPQ_SOURCE_DIR}/common/psprintf.c" - "${LIBPQ_SOURCE_DIR}/common/encnames.c" - "${LIBPQ_SOURCE_DIR}/common/logging.c" - - "${LIBPQ_SOURCE_DIR}/port/snprintf.c" - "${LIBPQ_SOURCE_DIR}/port/strlcpy.c" - "${LIBPQ_SOURCE_DIR}/port/strerror.c" - "${LIBPQ_SOURCE_DIR}/port/inet_net_ntop.c" - "${LIBPQ_SOURCE_DIR}/port/getpeereid.c" - "${LIBPQ_SOURCE_DIR}/port/chklocale.c" - "${LIBPQ_SOURCE_DIR}/port/noblock.c" - "${LIBPQ_SOURCE_DIR}/port/pg_strong_random.c" - "${LIBPQ_SOURCE_DIR}/port/pgstrcasecmp.c" - "${LIBPQ_SOURCE_DIR}/port/thread.c" - "${LIBPQ_SOURCE_DIR}/port/path.c" - ) - -add_library(_libpq ${SRCS}) - -add_definitions(-DHAVE_BIO_METH_NEW) -add_definitions(-DHAVE_HMAC_CTX_NEW) -add_definitions(-DHAVE_HMAC_CTX_FREE) - -target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR}) -target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_SOURCE_DIR}/include") -target_include_directories (_libpq SYSTEM PRIVATE "${LIBPQ_SOURCE_DIR}/configs") - -# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped -# for different OS'es like for jemalloc, not one generic for all OS'es like -# now. -if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL) - target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1) -endif() - -target_link_libraries (_libpq PRIVATE OpenSSL::SSL) - -add_library(ch_contrib::libpq ALIAS _libpq) diff --git a/contrib/libpqxx-cmake/CMakeLists.txt b/contrib/libpqxx-cmake/CMakeLists.txt index a3317404f95..18f19ebf0f1 100644 --- a/contrib/libpqxx-cmake/CMakeLists.txt +++ b/contrib/libpqxx-cmake/CMakeLists.txt @@ -1,10 +1,3 @@ -option(ENABLE_LIBPQXX "Enalbe libpqxx" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_LIBPQXX) - message(STATUS "Not using libpqxx") - return() -endif() - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpqxx") set (SRCS diff --git a/contrib/postgres b/contrib/postgres new file mode 160000 index 00000000000..665ff8c164d --- /dev/null +++ b/contrib/postgres @@ -0,0 +1 @@ +Subproject commit 665ff8c164d56d012e359735efe4d400c0564b44 diff --git a/contrib/postgres-cmake/CMakeLists.txt b/contrib/postgres-cmake/CMakeLists.txt new file mode 100644 index 00000000000..644e6530bbd --- /dev/null +++ b/contrib/postgres-cmake/CMakeLists.txt @@ -0,0 +1,78 @@ +# Build description for libpq which is part of the PostgreSQL sources + +set(POSTGRES_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/postgres") +set(LIBPQ_SOURCE_DIR "${POSTGRES_SOURCE_DIR}/src/interfaces/libpq") +set(LIBPQ_CMAKE_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/postgres-cmake") + +set(SRCS + "${LIBPQ_SOURCE_DIR}/fe-auth.c" + "${LIBPQ_SOURCE_DIR}/fe-auth-scram.c" + "${LIBPQ_SOURCE_DIR}/fe-connect.c" + "${LIBPQ_SOURCE_DIR}/fe-exec.c" + "${LIBPQ_SOURCE_DIR}/fe-lobj.c" + "${LIBPQ_SOURCE_DIR}/fe-misc.c" + "${LIBPQ_SOURCE_DIR}/fe-print.c" + "${LIBPQ_SOURCE_DIR}/fe-trace.c" + "${LIBPQ_SOURCE_DIR}/fe-protocol3.c" + "${LIBPQ_SOURCE_DIR}/fe-secure.c" + "${LIBPQ_SOURCE_DIR}/fe-secure-common.c" + "${LIBPQ_SOURCE_DIR}/fe-secure-openssl.c" + "${LIBPQ_SOURCE_DIR}/legacy-pqsignal.c" + "${LIBPQ_SOURCE_DIR}/libpq-events.c" + "${LIBPQ_SOURCE_DIR}/pqexpbuffer.c" + + "${POSTGRES_SOURCE_DIR}/src/common/scram-common.c" + "${POSTGRES_SOURCE_DIR}/src/common/sha2.c" + "${POSTGRES_SOURCE_DIR}/src/common/sha1.c" + "${POSTGRES_SOURCE_DIR}/src/common/md5.c" + "${POSTGRES_SOURCE_DIR}/src/common/md5_common.c" + "${POSTGRES_SOURCE_DIR}/src/common/hmac_openssl.c" + "${POSTGRES_SOURCE_DIR}/src/common/cryptohash.c" + "${POSTGRES_SOURCE_DIR}/src/common/saslprep.c" + "${POSTGRES_SOURCE_DIR}/src/common/unicode_norm.c" + "${POSTGRES_SOURCE_DIR}/src/common/ip.c" + "${POSTGRES_SOURCE_DIR}/src/common/jsonapi.c" + "${POSTGRES_SOURCE_DIR}/src/common/wchar.c" + "${POSTGRES_SOURCE_DIR}/src/common/base64.c" + "${POSTGRES_SOURCE_DIR}/src/common/link-canary.c" + "${POSTGRES_SOURCE_DIR}/src/common/fe_memutils.c" + "${POSTGRES_SOURCE_DIR}/src/common/string.c" + "${POSTGRES_SOURCE_DIR}/src/common/pg_get_line.c" + "${POSTGRES_SOURCE_DIR}/src/common/stringinfo.c" + "${POSTGRES_SOURCE_DIR}/src/common/psprintf.c" + "${POSTGRES_SOURCE_DIR}/src/common/encnames.c" + "${POSTGRES_SOURCE_DIR}/src/common/logging.c" + + "${POSTGRES_SOURCE_DIR}/src/port/snprintf.c" + "${POSTGRES_SOURCE_DIR}/src/port/strlcpy.c" + "${POSTGRES_SOURCE_DIR}/src/port/strerror.c" + "${POSTGRES_SOURCE_DIR}/src/port/inet_net_ntop.c" + "${POSTGRES_SOURCE_DIR}/src/port/getpeereid.c" + "${POSTGRES_SOURCE_DIR}/src/port/chklocale.c" + "${POSTGRES_SOURCE_DIR}/src/port/noblock.c" + "${POSTGRES_SOURCE_DIR}/src/port/pg_strong_random.c" + "${POSTGRES_SOURCE_DIR}/src/port/pgstrcasecmp.c" + "${POSTGRES_SOURCE_DIR}/src/port/thread.c" + "${POSTGRES_SOURCE_DIR}/src/port/path.c" +) + +add_library(_libpq ${SRCS}) + +add_definitions(-DHAVE_BIO_METH_NEW) +add_definitions(-DHAVE_HMAC_CTX_NEW) +add_definitions(-DHAVE_HMAC_CTX_FREE) + +target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR}) +target_include_directories (_libpq SYSTEM PUBLIC "${POSTGRES_SOURCE_DIR}/src/include") +target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_CMAKE_SOURCE_DIR}") # pre-generated headers + +# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped +# for different OS'es like for jemalloc, not one generic for all OS'es like +# now. +if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL) + target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1) +endif() + +target_link_libraries (_libpq PRIVATE OpenSSL::SSL) + +add_library(ch_contrib::libpq ALIAS _libpq) diff --git a/contrib/postgres-cmake/pg_config.h b/contrib/postgres-cmake/pg_config.h new file mode 100644 index 00000000000..ce16eab2239 --- /dev/null +++ b/contrib/postgres-cmake/pg_config.h @@ -0,0 +1,941 @@ +/* src/include/pg_config.h. Generated from pg_config.h.in by configure. */ +/* src/include/pg_config.h.in. Generated from configure.in by autoheader. */ + +/* Define to the type of arg 1 of 'accept' */ +#define ACCEPT_TYPE_ARG1 int + +/* Define to the type of arg 2 of 'accept' */ +#define ACCEPT_TYPE_ARG2 struct sockaddr * + +/* Define to the type of arg 3 of 'accept' */ +#define ACCEPT_TYPE_ARG3 size_t + +/* Define to the return type of 'accept' */ +#define ACCEPT_TYPE_RETURN int + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* The normal alignment of `double', in bytes. */ +#define ALIGNOF_DOUBLE 4 + +/* The normal alignment of `int', in bytes. */ +#define ALIGNOF_INT 4 + +/* The normal alignment of `long', in bytes. */ +#define ALIGNOF_LONG 4 + +/* The normal alignment of `long long int', in bytes. */ +#define ALIGNOF_LONG_LONG_INT 4 + +/* The normal alignment of `short', in bytes. */ +#define ALIGNOF_SHORT 2 + +/* Size of a disk block --- this also limits the size of a tuple. You can set + it bigger if you need bigger tuples (although TOAST should reduce the need + to have large tuples, since fields can be spread across multiple tuples). + BLCKSZ must be a power of 2. The maximum possible value of BLCKSZ is + currently 2^15 (32768). This is determined by the 15-bit widths of the + lp_off and lp_len fields in ItemIdData (see include/storage/itemid.h). + Changing BLCKSZ requires an initdb. */ +#define BLCKSZ 8192 + +/* Define to the default TCP port number on which the server listens and to + which clients will try to connect. This can be overridden at run-time, but + it's convenient if your clients have the right default compiled in. + (--with-pgport=PORTNUM) */ +#define DEF_PGPORT 5432 + +/* Define to the default TCP port number as a string constant. */ +#define DEF_PGPORT_STR "5432" + +/* Define to build with GSSAPI support. (--with-gssapi) */ +//#define ENABLE_GSS 0 + +/* Define to 1 if you want National Language Support. (--enable-nls) */ +/* #undef ENABLE_NLS */ + +/* Define to 1 to build client libraries as thread-safe code. + (--enable-thread-safety) */ +#define ENABLE_THREAD_SAFETY 1 + +/* Define to nothing if C supports flexible array members, and to 1 if it does + not. That way, with a declaration like `struct s { int n; double + d[FLEXIBLE_ARRAY_MEMBER]; };', the struct hack can be used with pre-C99 + compilers. When computing the size of such an object, don't use 'sizeof + (struct s)' as it overestimates the size. Use 'offsetof (struct s, d)' + instead. Don't use 'offsetof (struct s, d[0])', as this doesn't work with + MSVC and with C++ compilers. */ +#define FLEXIBLE_ARRAY_MEMBER /**/ + +/* float4 values are passed by value if 'true', by reference if 'false' */ +#define FLOAT4PASSBYVAL true + +/* float8, int8, and related values are passed by value if 'true', by + reference if 'false' */ +#define FLOAT8PASSBYVAL false + +/* Define to 1 if gettimeofday() takes only 1 argument. */ +/* #undef GETTIMEOFDAY_1ARG */ + +#ifdef GETTIMEOFDAY_1ARG +# define gettimeofday(a,b) gettimeofday(a) +#endif + +/* Define to 1 if you have the `append_history' function. */ +/* #undef HAVE_APPEND_HISTORY */ + +/* Define to 1 if you want to use atomics if available. */ +#define HAVE_ATOMICS 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_ATOMIC_H */ + +/* Define to 1 if you have the `cbrt' function. */ +#define HAVE_CBRT 1 + +/* Define to 1 if you have the `class' function. */ +/* #undef HAVE_CLASS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRTDEFS_H */ + +/* Define to 1 if you have the `crypt' function. */ +#define HAVE_CRYPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_CRYPT_H 1 + +/* Define to 1 if you have the declaration of `fdatasync', and to 0 if you + don't. */ +#define HAVE_DECL_FDATASYNC 1 + +/* Define to 1 if you have the declaration of `F_FULLFSYNC', and to 0 if you + don't. */ +#define HAVE_DECL_F_FULLFSYNC 0 + +/* Define to 1 if you have the declaration of `posix_fadvise', and to 0 if you + don't. */ +#define HAVE_DECL_POSIX_FADVISE 1 + +/* Define to 1 if you have the declaration of `snprintf', and to 0 if you + don't. */ +#define HAVE_DECL_SNPRINTF 1 + +/* Define to 1 if you have the declaration of `strlcat', and to 0 if you + don't. */ +#if OS_DARWIN +#define HAVE_DECL_STRLCAT 1 +#endif + +/* Define to 1 if you have the declaration of `strlcpy', and to 0 if you + don't. */ +#if OS_DARWIN +#define HAVE_DECL_STRLCPY 1 +#endif + +/* Define to 1 if you have the declaration of `sys_siglist', and to 0 if you + don't. */ +#define HAVE_DECL_SYS_SIGLIST 1 + +/* Define to 1 if you have the declaration of `vsnprintf', and to 0 if you + don't. */ +#define HAVE_DECL_VSNPRINTF 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DLD_H */ + +/* Define to 1 if you have the `dlopen' function. */ +#define HAVE_DLOPEN 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_EDITLINE_HISTORY_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_EDITLINE_READLINE_H 1 + +/* Define to 1 if you have the `fdatasync' function. */ +#define HAVE_FDATASYNC 1 + +/* Define to 1 if you have the `fls' function. */ +/* #undef HAVE_FLS */ + +/* Define to 1 if you have the `fpclass' function. */ +/* #undef HAVE_FPCLASS */ + +/* Define to 1 if you have the `fp_class' function. */ +/* #undef HAVE_FP_CLASS */ + +/* Define to 1 if you have the `fp_class_d' function. */ +/* #undef HAVE_FP_CLASS_D */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FP_CLASS_H */ + +/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ +#define HAVE_FSEEKO 1 + +/* Define to 1 if your compiler understands __func__. */ +#define HAVE_FUNCNAME__FUNC 1 + +/* Define to 1 if your compiler understands __FUNCTION__. */ +/* #undef HAVE_FUNCNAME__FUNCTION */ + +/* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */ +/* #undef HAVE_GCC__ATOMIC_INT32_CAS */ + +/* Define to 1 if you have __atomic_compare_exchange_n(int64 *, int *, int64). + */ +/* #undef HAVE_GCC__ATOMIC_INT64_CAS */ + +/* Define to 1 if you have __sync_lock_test_and_set(char *) and friends. */ +#define HAVE_GCC__SYNC_CHAR_TAS 1 + +/* Define to 1 if you have __sync_compare_and_swap(int *, int, int). */ +/* #undef HAVE_GCC__SYNC_INT32_CAS */ + +/* Define to 1 if you have __sync_lock_test_and_set(int *) and friends. */ +#define HAVE_GCC__SYNC_INT32_TAS 1 + +/* Define to 1 if you have __sync_compare_and_swap(int64 *, int64, int64). */ +/* #undef HAVE_GCC__SYNC_INT64_CAS */ + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `getifaddrs' function. */ +#define HAVE_GETIFADDRS 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getopt_long' function. */ +#define HAVE_GETOPT_LONG 1 + +/* Define to 1 if you have the `getpeereid' function. */ +/* #undef HAVE_GETPEEREID */ + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwuid_r' function. */ +#define HAVE_GETPWUID_R 1 + +/* Define to 1 if you have the `getrlimit' function. */ +#define HAVE_GETRLIMIT 1 + +/* Define to 1 if you have the `getrusage' function. */ +#define HAVE_GETRUSAGE 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +/* #undef HAVE_GETTIMEOFDAY */ + +/* Define to 1 if you have the header file. */ +//#define HAVE_GSSAPI_GSSAPI_H 0 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GSSAPI_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_HISTORY_H */ + +/* Define to 1 if you have the `history_truncate_file' function. */ +#define HAVE_HISTORY_TRUNCATE_FILE 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IEEEFP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_IFADDRS_H 1 + +/* Define to 1 if you have the `inet_aton' function. */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if the system has the type `int64'. */ +/* #undef HAVE_INT64 */ + +/* Define to 1 if the system has the type `int8'. */ +/* #undef HAVE_INT8 */ + +/* Define to 1 if the system has the type `intptr_t'. */ +#define HAVE_INTPTR_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the global variable 'int opterr'. */ +#define HAVE_INT_OPTERR 1 + +/* Define to 1 if you have the global variable 'int optreset'. */ +/* #undef HAVE_INT_OPTRESET */ + +/* Define to 1 if you have the global variable 'int timezone'. */ +#define HAVE_INT_TIMEZONE 1 + +/* Define to 1 if you have support for IPv6. */ +#define HAVE_IPV6 1 + +/* Define to 1 if you have isinf(). */ +#define HAVE_ISINF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LANGINFO_H 1 + +/* Define to 1 if you have the header file. */ +//#define HAVE_LDAP_H 0 + +/* Define to 1 if you have the `crypto' library (-lcrypto). */ +#define HAVE_LIBCRYPTO 1 + +/* Define to 1 if you have the `ldap' library (-lldap). */ +//#define HAVE_LIBLDAP 0 + +/* Define to 1 if you have the `m' library (-lm). */ +#define HAVE_LIBM 1 + +/* Define to 1 if you have the `pam' library (-lpam). */ +#define HAVE_LIBPAM 1 + +/* Define if you have a function readline library */ +#define HAVE_LIBREADLINE 1 + +/* Define to 1 if you have the `selinux' library (-lselinux). */ +/* #undef HAVE_LIBSELINUX */ + +/* Define to 1 if you have the `ssl' library (-lssl). */ +#define HAVE_LIBSSL 0 + +/* Define to 1 if you have the `wldap32' library (-lwldap32). */ +/* #undef HAVE_LIBWLDAP32 */ + +/* Define to 1 if you have the `xml2' library (-lxml2). */ +#define HAVE_LIBXML2 1 + +/* Define to 1 if you have the `xslt' library (-lxslt). */ +#define HAVE_LIBXSLT 1 + +/* Define to 1 if you have the `z' library (-lz). */ +#define HAVE_LIBZ 1 + +/* Define to 1 if constants of type 'long long int' should have the suffix LL. + */ +#define HAVE_LL_CONSTANTS 1 + +/* Define to 1 if the system has the type `locale_t'. */ +#define HAVE_LOCALE_T 1 + +/* Define to 1 if `long int' works and is 64 bits. */ +/* #undef HAVE_LONG_INT_64 */ + +/* Define to 1 if the system has the type `long long int'. */ +#define HAVE_LONG_LONG_INT 1 + +/* Define to 1 if `long long int' works and is 64 bits. */ +#define HAVE_LONG_LONG_INT_64 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MBARRIER_H */ + +/* Define to 1 if you have the `mbstowcs_l' function. */ +/* #undef HAVE_MBSTOWCS_L */ + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if the system has the type `MINIDUMP_TYPE'. */ +/* #undef HAVE_MINIDUMP_TYPE */ + +/* Define to 1 if you have the `mkdtemp' function. */ +#define HAVE_MKDTEMP 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_OSSP_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PAM_PAM_APPL_H */ + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the `posix_fadvise' function. */ +#define HAVE_POSIX_FADVISE 1 + +/* Define to 1 if you have the declaration of `preadv', and to 0 if you don't. */ +/* #undef HAVE_DECL_PREADV */ + +/* Define to 1 if you have the declaration of `pwritev', and to 0 if you don't. */ +/* #define HAVE_DECL_PWRITEV */ + +/* Define to 1 if you have the `X509_get_signature_info' function. */ +/* #undef HAVE_X509_GET_SIGNATURE_INFO */ + +/* Define to 1 if you have the POSIX signal interface. */ +#define HAVE_POSIX_SIGNALS 1 + +/* Define to 1 if the assembler supports PPC's LWARX mutex hint bit. */ +/* #undef HAVE_PPC_LWARX_MUTEX_HINT */ + +/* Define to 1 if you have the `pstat' function. */ +/* #undef HAVE_PSTAT */ + +/* Define to 1 if the PS_STRINGS thing exists. */ +/* #undef HAVE_PS_STRINGS */ + +/* Define to 1 if you have the `pthread_is_threaded_np' function. */ +/* #undef HAVE_PTHREAD_IS_THREADED_NP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `random' function. */ +#define HAVE_RANDOM 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_READLINE_HISTORY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_READLINE_READLINE_H */ + +/* Define to 1 if you have the `readlink' function. */ +#define HAVE_READLINK 1 + +/* Define to 1 if you have the `rint' function. */ +#define HAVE_RINT 1 + +/* Define to 1 if you have the global variable + 'rl_completion_append_character'. */ +/* #undef HAVE_RL_COMPLETION_APPEND_CHARACTER */ + +/* Define to 1 if you have the `rl_completion_matches' function. */ +#define HAVE_RL_COMPLETION_MATCHES 1 + +/* Define to 1 if you have the `rl_filename_completion_function' function. */ +#define HAVE_RL_FILENAME_COMPLETION_FUNCTION 1 + +/* Define to 1 if you have the `rl_reset_screen_size' function. */ +/* #undef HAVE_RL_RESET_SCREEN_SIZE */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SECURITY_PAM_APPL_H 1 + +/* Define to 1 if you have the `setproctitle' function. */ +/* #undef HAVE_SETPROCTITLE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `shm_open' function. */ +#define HAVE_SHM_OPEN 1 + +/* Define to 1 if you have the `sigprocmask' function. */ +#define HAVE_SIGPROCMASK 1 + +/* Define to 1 if you have sigsetjmp(). */ +#define HAVE_SIGSETJMP 1 + +/* Define to 1 if the system has the type `sig_atomic_t'. */ +#define HAVE_SIG_ATOMIC_T 1 + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* Define to 1 if you have spinlocks. */ +#define HAVE_SPINLOCKS 1 + +/* Define to 1 if you have the `srandom' function. */ +#define HAVE_SRANDOM 1 + +/* Define to 1 if you have the `SSL_CTX_set_num_tickets' function. */ +/* #define HAVE_SSL_CTX_SET_NUM_TICKETS */ + +/* Define to 1 if you have the `SSL_get_current_compression' function. */ +#define HAVE_SSL_GET_CURRENT_COMPRESSION 0 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the header file. */ +//#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +/* #undef HAVE_STRLCAT */ + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef HAVE_STRLCPY */ + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +#if (!OS_DARWIN) +#define HAVE_STRCHRNUL 1 +#endif + +/* Define to 1 if you have the `strtoq' function. */ +/* #undef HAVE_STRTOQ */ + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +/* #undef HAVE_STRTOUQ */ + +/* Define to 1 if the system has the type `struct addrinfo'. */ +#define HAVE_STRUCT_ADDRINFO 1 + +/* Define to 1 if the system has the type `struct cmsgcred'. */ +/* #undef HAVE_STRUCT_CMSGCRED */ + +/* Define to 1 if the system has the type `struct option'. */ +#define HAVE_STRUCT_OPTION 1 + +/* Define to 1 if `sa_len' is a member of `struct sockaddr'. */ +/* #undef HAVE_STRUCT_SOCKADDR_SA_LEN */ + +/* Define to 1 if the system has the type `struct sockaddr_storage'. */ +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 + +/* Define to 1 if `ss_family' is a member of `struct sockaddr_storage'. */ +#define HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY 1 + +/* Define to 1 if `ss_len' is a member of `struct sockaddr_storage'. */ +/* #undef HAVE_STRUCT_SOCKADDR_STORAGE_SS_LEN */ + +/* Define to 1 if `__ss_family' is a member of `struct sockaddr_storage'. */ +/* #undef HAVE_STRUCT_SOCKADDR_STORAGE___SS_FAMILY */ + +/* Define to 1 if `__ss_len' is a member of `struct sockaddr_storage'. */ +/* #undef HAVE_STRUCT_SOCKADDR_STORAGE___SS_LEN */ + +/* Define to 1 if `tm_zone' is a member of `struct tm'. */ +#define HAVE_STRUCT_TM_TM_ZONE 1 + +/* Define to 1 if you have the `symlink' function. */ +#define HAVE_SYMLINK 1 + +/* Define to 1 if you have the `sync_file_range' function. */ +/* #undef HAVE_SYNC_FILE_RANGE */ + +/* Define to 1 if you have the syslog interface. */ +#define HAVE_SYSLOG 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IPC_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PERSONALITY_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PSTAT_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SEM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SHM_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_SIGNALFD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_SOCKIO_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_TAS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#if (OS_DARWIN || OS_FREEBSD) +#define HAVE_SYS_UCRED_H 1 +#endif + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 +#define _GNU_SOURCE 1 /* Needed for glibc struct ucred */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* Define to 1 if your `struct tm' has `tm_zone'. Deprecated, use + `HAVE_STRUCT_TM_TM_ZONE' instead. */ +#define HAVE_TM_ZONE 1 + +/* Define to 1 if you have the `towlower' function. */ +#define HAVE_TOWLOWER 1 + +/* Define to 1 if you have the external array `tzname'. */ +#define HAVE_TZNAME 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UCRED_H */ + +/* Define to 1 if the system has the type `uint64'. */ +/* #undef HAVE_UINT64 */ + +/* Define to 1 if the system has the type `uint8'. */ +/* #undef HAVE_UINT8 */ + +/* Define to 1 if the system has the type `uintptr_t'. */ +#define HAVE_UINTPTR_T 1 + +/* Define to 1 if the system has the type `union semun'. */ +/* #undef HAVE_UNION_SEMUN */ + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have unix sockets. */ +#define HAVE_UNIX_SOCKETS 1 + +/* Define to 1 if you have the `unsetenv' function. */ +#define HAVE_UNSETENV 1 + +/* Define to 1 if the system has the type `unsigned long long int'. */ +#define HAVE_UNSIGNED_LONG_LONG_INT 1 + +/* Define to 1 if you have the `utime' function. */ +#define HAVE_UTIME 1 + +/* Define to 1 if you have the `utimes' function. */ +#define HAVE_UTIMES 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* Define to 1 if you have BSD UUID support. */ +/* #undef HAVE_UUID_BSD */ + +/* Define to 1 if you have E2FS UUID support. */ +/* #undef HAVE_UUID_E2FS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_UUID_H 1 + +/* Define to 1 if you have OSSP UUID support. */ +#define HAVE_UUID_OSSP 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_WCHAR_H 1 + +/* Define to 1 if you have the `wcstombs' function. */ +#define HAVE_WCSTOMBS 1 + +/* Define to 1 if you have the `wcstombs_l' function. */ +/* #undef HAVE_WCSTOMBS_L */ + +/* Define to 1 if you have the header file. */ +#define HAVE_WCTYPE_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINLDAP_H */ + +/* Define to 1 if your compiler understands __builtin_bswap32. */ +/* #undef HAVE__BUILTIN_BSWAP32 */ + +/* Define to 1 if your compiler understands __builtin_constant_p. */ +#define HAVE__BUILTIN_CONSTANT_P 1 + +/* Define to 1 if your compiler understands __builtin_frame_address. */ +/* #undef HAVE__BUILTIN_FRAME_ADDRESS */ + +/* Define to 1 if your compiler understands __builtin_types_compatible_p. */ +#define HAVE__BUILTIN_TYPES_COMPATIBLE_P 1 + +/* Define to 1 if your compiler understands __builtin_unreachable. */ +/* #undef HAVE__BUILTIN_UNREACHABLE */ + +/* Define to 1 if you have __cpuid. */ +/* #undef HAVE__CPUID */ + +/* Define to 1 if you have __get_cpuid. */ +/* #undef HAVE__GET_CPUID */ + +/* Define to 1 if your compiler understands _Static_assert. */ +/* #undef HAVE__STATIC_ASSERT */ + +/* Define to 1 if your compiler understands __VA_ARGS__ in macros. */ +#define HAVE__VA_ARGS 1 + +/* Define to the appropriate snprintf length modifier for 64-bit ints. */ +#define INT64_MODIFIER "ll" + +/* Define to 1 if `locale_t' requires . */ +/* #undef LOCALE_T_IN_XLOCALE */ + +/* Define as the maximum alignment requirement of any C data type. */ +#define MAXIMUM_ALIGNOF 4 + +/* Define bytes to use libc memset(). */ +#define MEMSET_LOOP_LIMIT 1024 + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "pgsql-bugs@postgresql.org" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "PostgreSQL" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "PostgreSQL 9.5.4" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "postgresql" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "9.5.4" + +/* Define to the name of a signed 128-bit integer type. */ +/* #undef PG_INT128_TYPE */ + +/* Define to the name of a signed 64-bit integer type. */ +#define PG_INT64_TYPE long long int + +/* Define to the name of the default PostgreSQL service principal in Kerberos + (GSSAPI). (--with-krb-srvnam=NAME) */ +#define PG_KRB_SRVNAM "postgres" + +/* PostgreSQL major version as a string */ +#define PG_MAJORVERSION "9.5" + +/* Define to gnu_printf if compiler supports it, else printf. */ +#define PG_PRINTF_ATTRIBUTE printf + +/* Define to 1 if "static inline" works without unwanted warnings from + compilations where static inline functions are defined but not called. */ +#define PG_USE_INLINE 1 + +/* PostgreSQL version as a string */ +#define PG_VERSION "9.5.4" + +/* PostgreSQL version as a number */ +#define PG_VERSION_NUM 90504 + +/* A string containing the version number, platform, and C compiler */ +#define PG_VERSION_STR "PostgreSQL 9.5.4 on i686-pc-linux-gnu, compiled by gcc (GCC) 4.1.2 20080704 (Red Hat 4.1.2-55), 32-bit" + +/* Define to 1 to allow profiling output to be saved separately for each + process. */ +/* #undef PROFILE_PID_DIR */ + +/* RELSEG_SIZE is the maximum number of blocks allowed in one disk file. Thus, + the maximum size of a single file is RELSEG_SIZE * BLCKSZ; relations bigger + than that are divided into multiple files. RELSEG_SIZE * BLCKSZ must be + less than your OS' limit on file size. This is often 2 GB or 4GB in a + 32-bit operating system, unless you have large file support enabled. By + default, we make the limit 1 GB to avoid any possible integer-overflow + problems within the OS. A limit smaller than necessary only means we divide + a large relation into more chunks than necessary, so it seems best to err + in the direction of a small limit. A power-of-2 value is recommended to + save a few cycles in md.c, but is not absolutely required. Changing + RELSEG_SIZE requires an initdb. */ +#define RELSEG_SIZE 131072 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 4 + +/* The size of `off_t', as computed by sizeof. */ +#define SIZEOF_OFF_T 8 + +/* The size of `size_t', as computed by sizeof. */ +#define SIZEOF_SIZE_T 4 + +/* The size of `void *', as computed by sizeof. */ +#define SIZEOF_VOID_P 4 + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if strerror_r() returns a int. */ +/* #undef STRERROR_R_INT */ + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* Define to 1 to build with assertion checks. (--enable-cassert) */ +/* #undef USE_ASSERT_CHECKING */ + +/* Define to 1 to build with Bonjour support. (--with-bonjour) */ +/* #undef USE_BONJOUR */ + +/* Define to 1 if you want float4 values to be passed by value. + (--enable-float4-byval) */ +#define USE_FLOAT4_BYVAL 1 + +/* Define to 1 if you want float8, int8, etc values to be passed by value. + (--enable-float8-byval) */ +/* #undef USE_FLOAT8_BYVAL */ + +/* Define to 1 if you want 64-bit integer timestamp and interval support. + (--enable-integer-datetimes) */ +#define USE_INTEGER_DATETIMES 1 + +/* Define to 1 to build with LDAP support. (--with-ldap) */ +//#define USE_LDAP 0 + +/* Define to 1 to build with XML support. (--with-libxml) */ +#define USE_LIBXML 1 + +/* Define to 1 to use XSLT support when building contrib/xml2. + (--with-libxslt) */ +#define USE_LIBXSLT 1 + +/* Define to select named POSIX semaphores. */ +/* #undef USE_NAMED_POSIX_SEMAPHORES */ + +/* Define to build with OpenSSL support. (--with-openssl) */ +#define USE_OPENSSL 0 + +#define USE_OPENSSL_RANDOM 0 + +#define FRONTEND 1 + +/* Define to 1 to build with PAM support. (--with-pam) */ +#define USE_PAM 1 + +/* Use replacement snprintf() functions. */ +/* #undef USE_REPL_SNPRINTF */ + +/* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ +#define USE_SLICING_BY_8_CRC32C 1 + +/* Define to 1 use Intel SSE 4.2 CRC instructions. */ +/* #undef USE_SSE42_CRC32C */ + +/* Define to 1 to use Intel SSSE 4.2 CRC instructions with a runtime check. */ +/* #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK */ + +/* Define to select SysV-style semaphores. */ +#define USE_SYSV_SEMAPHORES 1 + +/* Define to select SysV-style shared memory. */ +#define USE_SYSV_SHARED_MEMORY 1 + +/* Define to select unnamed POSIX semaphores. */ +/* #undef USE_UNNAMED_POSIX_SEMAPHORES */ + +/* Define to select Win32-style semaphores. */ +/* #undef USE_WIN32_SEMAPHORES */ + +/* Define to select Win32-style shared memory. */ +/* #undef USE_WIN32_SHARED_MEMORY */ + +/* Define to 1 if `wcstombs_l' requires . */ +/* #undef WCSTOMBS_L_IN_XLOCALE */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Size of a WAL file block. This need have no particular relation to BLCKSZ. + XLOG_BLCKSZ must be a power of 2, and if your system supports O_DIRECT I/O, + XLOG_BLCKSZ must be a multiple of the alignment requirement for direct-I/O + buffers, else direct I/O may fail. Changing XLOG_BLCKSZ requires an initdb. + */ +#define XLOG_BLCKSZ 8192 + +/* XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2 + and larger than XLOG_BLCKSZ (preferably, a great deal larger than + XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */ +#define XLOG_SEG_SIZE (16 * 1024 * 1024) + + + +/* Number of bits in a file offset, on hosts where this is settable. */ +#define _FILE_OFFSET_BITS 64 + +/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */ +/* #undef _LARGEFILE_SOURCE */ + +/* Define for large files, on AIX-style hosts. */ +/* #undef _LARGE_FILES */ + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +/* #undef inline */ +#endif + +/* Define to the type of a signed integer type wide enough to hold a pointer, + if such a type exists, and if the system does not define it. */ +/* #undef intptr_t */ + +/* Define to empty if the C compiler does not understand signed types. */ +/* #undef signed */ + +/* Define to the type of an unsigned integer type wide enough to hold a + pointer, if such a type exists, and if the system does not define it. */ +/* #undef uintptr_t */ diff --git a/contrib/postgres-cmake/pg_config_ext.h b/contrib/postgres-cmake/pg_config_ext.h new file mode 100644 index 00000000000..c5401355766 --- /dev/null +++ b/contrib/postgres-cmake/pg_config_ext.h @@ -0,0 +1,7 @@ +/* + * * src/include/pg_config_ext.h.in. This is generated manually, not by + * * autoheader, since we want to limit which symbols get defined here. + * */ + +/* Define to the name of a signed 64-bit integer type. */ +#define PG_INT64_TYPE long long int diff --git a/contrib/postgres-cmake/pg_config_os.h b/contrib/postgres-cmake/pg_config_os.h new file mode 100644 index 00000000000..ecfed0d7b19 --- /dev/null +++ b/contrib/postgres-cmake/pg_config_os.h @@ -0,0 +1,34 @@ +#if defined(OS_DARWIN) + +/* src/include/port/darwin.h */ +#define __darwin__ 1 + +#if HAVE_DECL_F_FULLFSYNC /* not present before macOS 10.3 */ +#define HAVE_FSYNC_WRITETHROUGH +#endif + +#else +/* src/include/port/linux.h */ +/* + * As of July 2007, all known versions of the Linux kernel will sometimes + * return EIDRM for a shmctl() operation when EINVAL is correct (it happens + * when the low-order 15 bits of the supplied shm ID match the slot number + * assigned to a newer shmem segment). We deal with this by assuming that + * EIDRM means EINVAL in PGSharedMemoryIsInUse(). This is reasonably safe + * since in fact Linux has no excuse for ever returning EIDRM; it doesn't + * track removed segments in a way that would allow distinguishing them from + * private ones. But someday that code might get upgraded, and we'd have + * to have a kernel version test here. + */ +#define HAVE_LINUX_EIDRM_BUG + +/* + * Set the default wal_sync_method to fdatasync. With recent Linux versions, + * xlogdefs.h's normal rules will prefer open_datasync, which (a) doesn't + * perform better and (b) causes outright failures on ext4 data=journal + * filesystems, because those don't support O_DIRECT. + */ +#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC + +#endif + diff --git a/contrib/postgres-cmake/pg_config_paths.h b/contrib/postgres-cmake/pg_config_paths.h new file mode 100644 index 00000000000..b366e3994b3 --- /dev/null +++ b/contrib/postgres-cmake/pg_config_paths.h @@ -0,0 +1,12 @@ +#define PGBINDIR "/bin" +#define PGSHAREDIR "/share" +#define SYSCONFDIR "/etc" +#define INCLUDEDIR "/include" +#define PKGINCLUDEDIR "/include" +#define INCLUDEDIRSERVER "/include/server" +#define LIBDIR "/lib" +#define PKGLIBDIR "/lib" +#define LOCALEDIR "/share/locale" +#define DOCDIR "/doc" +#define HTMLDIR "/doc" +#define MANDIR "/man" diff --git a/contrib/postgres-cmake/utils/errcodes.h b/contrib/postgres-cmake/utils/errcodes.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 79e809ea7f1..3102ab8297c 100755 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -109,7 +109,7 @@ if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CL ::/0 - ${CLICKHOUSE_PASSWORD} + /]]]]>}]]> default ${CLICKHOUSE_ACCESS_MANAGEMENT} diff --git a/docs/en/engines/database-engines/index.md b/docs/en/engines/database-engines/index.md index 233cbbb4247..fffd780e62a 100644 --- a/docs/en/engines/database-engines/index.md +++ b/docs/en/engines/database-engines/index.md @@ -13,16 +13,17 @@ Here is a complete list of available database engines. Follow the links for more - [Atomic](../../engines/database-engines/atomic.md) -- [MySQL](../../engines/database-engines/mysql.md) +- [Lazy](../../engines/database-engines/lazy.md) + +- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) - [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) -- [Lazy](../../engines/database-engines/lazy.md) +- [MySQL](../../engines/database-engines/mysql.md) - [PostgreSQL](../../engines/database-engines/postgresql.md) -- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) - - [Replicated](../../engines/database-engines/replicated.md) - [SQLite](../../engines/database-engines/sqlite.md) + diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index 3aa6dd01ea3..97185f35e1e 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -155,6 +155,12 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine. + Each table can have subset of replicated columns in brackets. If subset of columns is omitted, then all columns for table will be replicated. + + ``` sql + materialized_postgresql_tables_list = 'table1(co1, col2),table2,table3(co3, col5, col7) + ``` + Default value: empty list — means whole PostgreSQL database will be replicated. ### `materialized_postgresql_schema` {#materialized-postgresql-schema} diff --git a/docs/en/engines/table-engines/integrations/nats.md b/docs/en/engines/table-engines/integrations/nats.md index 78ce537224c..806437571f7 100644 --- a/docs/en/engines/table-engines/integrations/nats.md +++ b/docs/en/engines/table-engines/integrations/nats.md @@ -112,7 +112,7 @@ Example: ``` The NATS server configuration can be added using the ClickHouse config file. - More specifically you can add Redis password for NATS engine: +More specifically you can add Redis password for NATS engine: ``` xml @@ -167,7 +167,7 @@ If you want to change the target table by using `ALTER`, we recommend disabling - `_subject` - NATS message subject. Data type: `String`. -Additional virtual columns when `kafka_handle_error_mode='stream'`: +Additional virtual columns when `nats_handle_error_mode='stream'`: - `_raw_message` - Raw message that couldn't be parsed successfully. Data type: `Nullable(String)`. - `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`. diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index 3c75b8dbef0..f507e2b9f86 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -107,6 +107,10 @@ The vector similarity index currently does not work with per-table, non-default [here](https://github.com/ClickHouse/ClickHouse/pull/51325#issuecomment-1605920475)). If necessary, the value must be changed in config.xml. ::: +Vector index creation is known to be slow. To speed the process up, index creation can be parallelized. The maximum number of threads can be +configured using server configuration +setting [max_build_vector_similarity_index_thread_pool_size](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size). + ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write requests. diff --git a/docs/en/engines/table-engines/special/filelog.md b/docs/en/engines/table-engines/special/filelog.md index 82201053bc5..b9fb4ceaedf 100644 --- a/docs/en/engines/table-engines/special/filelog.md +++ b/docs/en/engines/table-engines/special/filelog.md @@ -97,7 +97,7 @@ If you want to change the target table by using `ALTER`, we recommend disabling - `_filename` - Name of the log file. Data type: `LowCardinality(String)`. - `_offset` - Offset in the log file. Data type: `UInt64`. -Additional virtual columns when `kafka_handle_error_mode='stream'`: +Additional virtual columns when `handle_error_mode='stream'`: - `_raw_record` - Raw record that couldn't be parsed successfully. Data type: `Nullable(String)`. - `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index df96b8129f1..2dd7c1dbfb6 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -826,17 +826,17 @@ Result: ## JSONAsObject {#jsonasobject} -In this format, a single JSON object is interpreted as a single [Object('json')](/docs/en/sql-reference/data-types/json.md) value. If the input has several JSON objects (comma separated), they are interpreted as separate rows. If the input data is enclosed in square brackets, it is interpreted as an array of JSONs. +In this format, a single JSON object is interpreted as a single [JSON](/docs/en/sql-reference/data-types/newjson.md) value. If the input has several JSON objects (comma separated), they are interpreted as separate rows. If the input data is enclosed in square brackets, it is interpreted as an array of JSONs. -This format can only be parsed for a table with a single field of type [Object('json')](/docs/en/sql-reference/data-types/json.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized). +This format can only be parsed for a table with a single field of type [JSON](/docs/en/sql-reference/data-types/newjson.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized). **Examples** Query: ``` sql -SET allow_experimental_object_type = 1; -CREATE TABLE json_as_object (json Object('json')) ENGINE = Memory; +SET allow_experimental_json_type = 1; +CREATE TABLE json_as_object (json JSON) ENGINE = Memory; INSERT INTO json_as_object (json) FORMAT JSONAsObject {"foo":{"bar":{"x":"y"},"baz":1}},{},{"any json stucture":1} SELECT * FROM json_as_object FORMAT JSONEachRow; ``` @@ -844,9 +844,9 @@ SELECT * FROM json_as_object FORMAT JSONEachRow; Result: ``` response -{"json":{"any json stucture":0,"foo":{"bar":{"x":"y"},"baz":1}}} -{"json":{"any json stucture":0,"foo":{"bar":{"x":""},"baz":0}}} -{"json":{"any json stucture":1,"foo":{"bar":{"x":""},"baz":0}}} +{"json":{"foo":{"bar":{"x":"y"},"baz":"1"}}} +{"json":{}} +{"json":{"any json stucture":"1"}} ``` **An array of JSON objects** @@ -854,35 +854,34 @@ Result: Query: ``` sql -SET allow_experimental_object_type = 1; -CREATE TABLE json_square_brackets (field Object('json')) ENGINE = Memory; +SET allow_experimental_json_type = 1; +CREATE TABLE json_square_brackets (field JSON) ENGINE = Memory; INSERT INTO json_square_brackets FORMAT JSONAsObject [{"id": 1, "name": "name1"}, {"id": 2, "name": "name2"}]; - SELECT * FROM json_square_brackets FORMAT JSONEachRow; ``` Result: ```response -{"field":{"id":1,"name":"name1"}} -{"field":{"id":2,"name":"name2"}} +{"field":{"id":"1","name":"name1"}} +{"field":{"id":"2","name":"name2"}} ``` **Columns with default values** ```sql -SET allow_experimental_object_type = 1; -CREATE TABLE json_as_object (json Object('json'), time DateTime MATERIALIZED now()) ENGINE = Memory; +SET allow_experimental_json_type = 1; +CREATE TABLE json_as_object (json JSON, time DateTime MATERIALIZED now()) ENGINE = Memory; INSERT INTO json_as_object (json) FORMAT JSONAsObject {"foo":{"bar":{"x":"y"},"baz":1}}; INSERT INTO json_as_object (json) FORMAT JSONAsObject {}; INSERT INTO json_as_object (json) FORMAT JSONAsObject {"any json stucture":1} -SELECT * FROM json_as_object FORMAT JSONEachRow +SELECT time, json FROM json_as_object FORMAT JSONEachRow ``` ```resonse -{"json":{"any json stucture":0,"foo":{"bar":{"x":"y"},"baz":1}},"time":"2024-07-25 17:02:45"} -{"json":{"any json stucture":0,"foo":{"bar":{"x":""},"baz":0}},"time":"2024-07-25 17:02:47"} -{"json":{"any json stucture":1,"foo":{"bar":{"x":""},"baz":0}},"time":"2024-07-25 17:02:50"} +{"time":"2024-09-16 12:18:10","json":{}} +{"time":"2024-09-16 12:18:13","json":{"any json stucture":"1"}} +{"time":"2024-09-16 12:18:08","json":{"foo":{"bar":{"x":"y"},"baz":"1"}}} ``` ## JSONCompact {#jsoncompact} @@ -1396,6 +1395,7 @@ SELECT * FROM json_each_row_nested - [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - ignore unknown keys in json object for named tuples. Default value - `false`. - [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`. - [input_format_json_throw_on_bad_escape_sequence](/docs/en/operations/settings/settings-formats.md/#input_format_json_throw_on_bad_escape_sequence) - throw an exception if JSON string contains bad escape sequence. If disabled, bad escape sequences will remain as is in the data. Default value - `true`. +- [input_format_json_empty_as_default](/docs/en/operations/settings/settings-formats.md/#input_format_json_empty_as_default) - treat empty fields in JSON input as default values. Default value - `false`. For complex default expressions [input_format_defaults_for_omitted_fields](/docs/en/operations/settings/settings-formats.md/#input_format_defaults_for_omitted_fields) must be enabled too. - [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`. - [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`. - [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`. diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md index 09fac45d7ae..a7514966fa7 100644 --- a/docs/en/operations/external-authenticators/ssl-x509.md +++ b/docs/en/operations/external-authenticators/ssl-x509.md @@ -6,7 +6,7 @@ import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.m -[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. +[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. `subjectAltName extension` supports the usage of one wildcard '*' in the server configuration. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. To enable SSL certificate authentication, a list of `Common Name`'s or `Subject Alt Name`'s for each ClickHouse user must be specified in the settings file `users.xml `: @@ -30,6 +30,12 @@ To enable SSL certificate authentication, a list of `Common Name`'s or `Subject + + + + URI:spiffe://foo.com/*/bar + + ``` diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index ccc8cf017ca..014141aa33b 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -491,6 +491,14 @@ Type: Double Default: 0.9 +## max_build_vector_similarity_index_thread_pool_size {#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size} + +The maximum number of threads to use for building vector indexes. 0 means all cores. + +Type: UInt64 + +Default: 16 + ## cgroups_memory_usage_observer_wait_time Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see @@ -3142,3 +3150,15 @@ Default value: "default" **See Also** - [Workload Scheduling](/docs/en/operations/workload-scheduling.md) + +## max_authentication_methods_per_user {#max_authentication_methods_per_user} + +The maximum number of authentication methods a user can be created with or altered to. +Changing this setting does not affect existing users. Create/alter authentication-related queries will fail if they exceed the limit specified in this setting. +Non authentication create/alter queries will succeed. + +Type: UInt64 + +Default value: 100 + +Zero means unlimited diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 5aad8db2809..c012d065574 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -752,6 +752,17 @@ Possible values: Default value: 0. +### input_format_json_empty_as_default {#input_format_json_empty_as_default} + +When enabled, replace empty input fields in JSON with default values. For complex default expressions `input_format_defaults_for_omitted_fields` must be enabled too. + +Possible values: + ++ 0 — Disable. ++ 1 — Enable. + +Default value: 0. + ## TSV format settings {#tsv-format-settings} ### input_format_tsv_empty_as_default {#input_format_tsv_empty_as_default} diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index fbfdd66d1a3..6f026766750 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -55,7 +55,7 @@ keeper foo bar - `touch ''` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists - `get ''` -- Returns the node's value - `rm '' [version]` -- Removes the node only if version matches (default: -1) -- `rmr ''` -- Recursively deletes path. Confirmation required +- `rmr '' [limit]` -- Recursively deletes path if the subtree size is smaller than the limit. Confirmation required (default limit = 100) - `flwc ` -- Executes four-letter-word command - `help` -- Prints this message - `get_direct_children_number '[path]'` -- Get numbers of direct children nodes under a specific path diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index 6216b83c2ef..c5c436b151b 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -12,9 +12,10 @@ Syntax: ``` sql ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}] + [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}] [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] + [RESET AUTHENTICATION METHODS TO NEW] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] @@ -62,3 +63,31 @@ Allows the user with `john` account to grant his privileges to the user with `ja ``` sql ALTER USER john GRANTEES jack; ``` + +Adds new authentication methods to the user while keeping the existing ones: + +``` sql +ALTER USER user1 ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' +``` + +Notes: +1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. +2. `no_password` can not co-exist with other authentication methods for security reasons. +Because of that, it is not possible to `ADD` a `no_password` authentication method. The below query will throw an error: + +``` sql +ALTER USER user1 ADD IDENTIFIED WITH no_password +``` + +If you want to drop authentication methods for a user and rely on `no_password`, you must specify in the below replacing form. + +Reset authentication methods and adds the ones specified in the query (effect of leading IDENTIFIED without the ADD keyword): + +``` sql +ALTER USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' +``` + +Reset authentication methods and keep the most recent added one: +``` sql +ALTER USER user1 RESET AUTHENTICATION METHODS TO NEW +``` diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 8c9143ee086..218589391a2 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -15,6 +15,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] + [RESET AUTHENTICATION METHODS TO NEW] [IN access_storage_type] [DEFAULT ROLE role [,...]] [DEFAULT DATABASE database | NONE] @@ -144,6 +145,17 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re The available password types are: `plaintext_password`, `sha256_password`, `double_sha1_password`. +7. Multiple authentication methods can be specified: + + ```sql + CREATE USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'' + ``` + +Notes: +1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. +2. `no_password` can not co-exist with other authentication methods for security reasons. Therefore, you can only specify +`no_password` if it is the only authentication method in the query. + ## User Host User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways: diff --git a/programs/keeper-client/Commands.cpp b/programs/keeper-client/Commands.cpp index 4ad2eb31e6d..b4a5329b01f 100644 --- a/programs/keeper-client/Commands.cpp +++ b/programs/keeper-client/Commands.cpp @@ -506,14 +506,23 @@ bool RMRCommand::parse(IParser::Pos & pos, std::shared_ptr & nod return false; node->args.push_back(std::move(path)); + ASTPtr remove_nodes_limit; + if (ParserUnsignedInteger{}.parse(pos, remove_nodes_limit, expected)) + node->args.push_back(remove_nodes_limit->as().value); + else + node->args.push_back(UInt64(100)); + return true; } void RMRCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const { String path = client->getAbsolutePath(query->args[0].safeGet()); + UInt64 remove_nodes_limit = query->args[1].safeGet(); + client->askConfirmation( - "You are going to recursively delete path " + path, [client, path] { client->zookeeper->removeRecursive(path); }); + "You are going to recursively delete path " + path, + [client, path, remove_nodes_limit] { client->zookeeper->removeRecursive(path, static_cast(remove_nodes_limit)); }); } bool ReconfigCommand::parse(IParser::Pos & pos, std::shared_ptr & node, DB::Expected & expected) const diff --git a/programs/keeper-client/Commands.h b/programs/keeper-client/Commands.h index 686a752b6b6..da577ce1e65 100644 --- a/programs/keeper-client/Commands.h +++ b/programs/keeper-client/Commands.h @@ -184,7 +184,7 @@ class RMRCommand : public IKeeperClientCommand void execute(const ASTKeeperQuery * query, KeeperClient * client) const override; - String getHelpMessage() const override { return "{} -- Recursively deletes path. Confirmation required"; } + String getHelpMessage() const override { return "{} [limit] -- Recursively deletes path if the subtree size is smaller than the limit. Confirmation required (default limit = 100)"; } }; class ReconfigCommand : public IKeeperClientCommand diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index d9ee89b45ce..e8ea21852b5 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -29,6 +29,7 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_RESTORE_TABLE; + extern const int ACCESS_ENTITY_ALREADY_EXISTS; extern const int LOGICAL_ERROR; } @@ -175,9 +176,46 @@ namespace return res; } - std::unordered_map resolveDependencies(const std::unordered_map> & dependencies, const AccessControl & access_control, bool allow_unresolved_dependencies) + /// Checks if new entities (which we're going to restore) already exist, + /// and either skips them or throws an exception depending on the restore settings. + void checkExistingEntities(std::vector> & entities, + std::unordered_map & old_to_new_id, + const AccessControl & access_control, + RestoreAccessCreationMode creation_mode) + { + if (creation_mode == RestoreAccessCreationMode::kReplace) + return; + + auto should_skip = [&](const std::pair & id_and_entity) + { + const auto & id = id_and_entity.first; + const auto & entity = *id_and_entity.second; + auto existing_id = access_control.find(entity.getType(), entity.getName()); + if (!existing_id) + { + return false; + } + else if (creation_mode == RestoreAccessCreationMode::kCreateIfNotExists) + { + old_to_new_id[id] = *existing_id; + return true; + } + else + { + throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists", entity.formatTypeWithName()); + } + }; + + std::erase_if(entities, should_skip); + } + + /// If new entities (which we're going to restore) depend on other entities which are not going to be restored or not present in the backup + /// then we should try to replace those dependencies with already existing entities. + void resolveDependencies(const std::unordered_map> & dependencies, + std::unordered_map & old_to_new_ids, + const AccessControl & access_control, + bool allow_unresolved_dependencies) { - std::unordered_map old_to_new_ids; for (const auto & [id, name_and_type] : dependencies) { std::optional new_id; @@ -188,9 +226,9 @@ namespace if (new_id) old_to_new_ids.emplace(id, *new_id); } - return old_to_new_ids; } + /// Generates random IDs for the new entities. void generateRandomIDs(std::vector> & entities, std::unordered_map & old_to_new_ids) { Poco::UUIDGenerator generator; @@ -203,27 +241,12 @@ namespace } } - void replaceDependencies(std::vector> & entities, const std::unordered_map & old_to_new_ids) + /// Updates dependencies of the new entities using a specified map. + void replaceDependencies(std::vector> & entities, + const std::unordered_map & old_to_new_ids) { for (auto & entity : entities | boost::adaptors::map_values) - { - bool need_replace = false; - for (const auto & dependency : entity->findDependencies()) - { - if (old_to_new_ids.contains(dependency)) - { - need_replace = true; - break; - } - } - - if (!need_replace) - continue; - - auto new_entity = entity->clone(); - new_entity->replaceDependencies(old_to_new_ids); - entity = new_entity; - } + IAccessEntity::replaceDependencies(entity, old_to_new_ids); } AccessRightsElements getRequiredAccessToRestore(const std::vector> & entities) @@ -314,7 +337,9 @@ std::pair makeBackupEntryForAccess( AccessRestorerFromBackup::AccessRestorerFromBackup( const BackupPtr & backup_, const RestoreSettings & restore_settings_) - : backup(backup_), allow_unresolved_access_dependencies(restore_settings_.allow_unresolved_access_dependencies) + : backup(backup_) + , creation_mode(restore_settings_.create_access) + , allow_unresolved_dependencies(restore_settings_.allow_unresolved_access_dependencies) { } @@ -362,7 +387,9 @@ std::vector> AccessRestorerFromBackup::getAcces { auto new_entities = entities; - auto old_to_new_ids = resolveDependencies(dependencies, access_control, allow_unresolved_access_dependencies); + std::unordered_map old_to_new_ids; + checkExistingEntities(new_entities, old_to_new_ids, access_control, creation_mode); + resolveDependencies(dependencies, old_to_new_ids, access_control, allow_unresolved_dependencies); generateRandomIDs(new_entities, old_to_new_ids); replaceDependencies(new_entities, old_to_new_ids); diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index aa59d6bf201..51a1112e5d5 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -17,6 +17,7 @@ using BackupPtr = std::shared_ptr; class IBackupEntry; using BackupEntryPtr = std::shared_ptr; struct RestoreSettings; +enum class RestoreAccessCreationMode : uint8_t; /// Makes a backup of access entities of a specified type. @@ -45,7 +46,8 @@ public: private: BackupPtr backup; - bool allow_unresolved_access_dependencies = false; + RestoreAccessCreationMode creation_mode; + bool allow_unresolved_dependencies = false; std::vector> entities; std::unordered_map> dependencies; std::unordered_set data_paths; diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 95a467bbbe5..ec513f0692d 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -544,9 +544,9 @@ scope_guard AccessControl::subscribeForChanges(const std::vector & ids, co return changes_notifier->subscribeForChanges(ids, handler); } -bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) +bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { - if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists)) + if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists, conflicting_id)) { changes_notifier->sendNotifications(); return true; diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index bfaf256ad48..0c3bb9352f0 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -243,7 +243,7 @@ private: class CustomSettingsPrefixes; class PasswordComplexityRules; - bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; + bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; diff --git a/src/Access/AccessEntityIO.cpp b/src/Access/AccessEntityIO.cpp index 1b073329296..cc1b7eee807 100644 --- a/src/Access/AccessEntityIO.cpp +++ b/src/Access/AccessEntityIO.cpp @@ -82,7 +82,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition) if (res) throw Exception(ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION, "Two access entities attached in the same file"); res = user = std::make_unique(); - InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true); + InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true, /* max_number_of_authentication_methods = zero is unlimited*/ 0); } else if (auto * create_role_query = query->as()) { diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 6b9a6e05cf6..8d5d04a4ed2 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -14,11 +14,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; - extern const int SUPPORT_IS_DISABLED; -} namespace { @@ -84,12 +79,140 @@ namespace return false; } #endif -} + bool checkKerberosAuthentication( + const GSSAcceptorContext * gss_acceptor_context, + const AuthenticationData & authentication_method, + const ExternalAuthenticators & external_authenticators) + { + return authentication_method.getType() == AuthenticationType::KERBEROS + && external_authenticators.checkKerberosCredentials(authentication_method.getKerberosRealm(), *gss_acceptor_context); + } + + bool checkMySQLAuthentication( + const MySQLNative41Credentials * mysql_credentials, + const AuthenticationData & authentication_method) + { + switch (authentication_method.getType()) + { + case AuthenticationType::PLAINTEXT_PASSWORD: + return checkPasswordPlainTextMySQL( + mysql_credentials->getScramble(), + mysql_credentials->getScrambledPassword(), + authentication_method.getPasswordHashBinary()); + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + return checkPasswordDoubleSHA1MySQL( + mysql_credentials->getScramble(), + mysql_credentials->getScrambledPassword(), + authentication_method.getPasswordHashBinary()); + default: + return false; + } + } + + bool checkBasicAuthentication( + const BasicCredentials * basic_credentials, + const AuthenticationData & authentication_method, + const ExternalAuthenticators & external_authenticators, + SettingsChanges & settings) + { + switch (authentication_method.getType()) + { + case AuthenticationType::NO_PASSWORD: + { + return true; // N.B. even if the password is not empty! + } + case AuthenticationType::PLAINTEXT_PASSWORD: + { + return checkPasswordPlainText(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary()); + } + case AuthenticationType::SHA256_PASSWORD: + { + return checkPasswordSHA256( + basic_credentials->getPassword(), authentication_method.getPasswordHashBinary(), authentication_method.getSalt()); + } + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + { + return checkPasswordDoubleSHA1(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary()); + } + case AuthenticationType::LDAP: + { + return external_authenticators.checkLDAPCredentials(authentication_method.getLDAPServerName(), *basic_credentials); + } + case AuthenticationType::BCRYPT_PASSWORD: + { + return checkPasswordBcrypt(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary()); + } + case AuthenticationType::HTTP: + { + if (authentication_method.getHTTPAuthenticationScheme() == HTTPAuthenticationScheme::BASIC) + { + return external_authenticators.checkHTTPBasicCredentials( + authentication_method.getHTTPAuthenticationServerName(), *basic_credentials, settings); + } + break; + } + default: + break; + } + + return false; + } + + bool checkSSLCertificateAuthentication( + const SSLCertificateCredentials * ssl_certificate_credentials, + const AuthenticationData & authentication_method) + { + if (AuthenticationType::SSL_CERTIFICATE != authentication_method.getType()) + { + return false; + } + + for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN}) + { + for (const auto & subject : authentication_method.getSSLCertificateSubjects().at(type)) + { + if (ssl_certificate_credentials->getSSLCertificateSubjects().at(type).contains(subject)) + return true; + + // Wildcard support (1 only) + if (subject.contains('*')) + { + auto prefix = std::string_view(subject).substr(0, subject.find('*')); + auto suffix = std::string_view(subject).substr(subject.find('*') + 1); + auto slashes = std::count(subject.begin(), subject.end(), '/'); + + for (const auto & certificate_subject : ssl_certificate_credentials->getSSLCertificateSubjects().at(type)) + { + bool matches_wildcard = certificate_subject.starts_with(prefix) && certificate_subject.ends_with(suffix); + + // '*' must not represent a '/' in URI, so check if the number of '/' are equal + bool matches_slashes = slashes == count(certificate_subject.begin(), certificate_subject.end(), '/'); + + if (matches_wildcard && matches_slashes) + return true; + } + } + } + } + + return false; + } + +#if USE_SSH + bool checkSshAuthentication( + const SshCredentials * ssh_credentials, + const AuthenticationData & authentication_method) + { + return AuthenticationType::SSH_KEY == authentication_method.getType() + && checkSshSignature(authentication_method.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal()); + } +#endif +} bool Authentication::areCredentialsValid( const Credentials & credentials, - const AuthenticationData & auth_data, + const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) { @@ -98,204 +221,35 @@ bool Authentication::areCredentialsValid( if (const auto * gss_acceptor_context = typeid_cast(&credentials)) { - switch (auth_data.getType()) - { - case AuthenticationType::NO_PASSWORD: - case AuthenticationType::PLAINTEXT_PASSWORD: - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::JWT: - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); - - case AuthenticationType::KERBEROS: - return external_authenticators.checkKerberosCredentials(auth_data.getKerberosRealm(), *gss_acceptor_context); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::MAX: - break; - } + return checkKerberosAuthentication(gss_acceptor_context, authentication_method, external_authenticators); } if (const auto * mysql_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) - { - case AuthenticationType::NO_PASSWORD: - return true; // N.B. even if the password is not empty! - - case AuthenticationType::PLAINTEXT_PASSWORD: - return checkPasswordPlainTextMySQL(mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - return checkPasswordDoubleSHA1MySQL(mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::KERBEROS: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::JWT: - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::MAX: - break; - } + return checkMySQLAuthentication(mysql_credentials, authentication_method); } if (const auto * basic_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) - { - case AuthenticationType::NO_PASSWORD: - return true; // N.B. even if the password is not empty! - - case AuthenticationType::PLAINTEXT_PASSWORD: - return checkPasswordPlainText(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::SHA256_PASSWORD: - return checkPasswordSHA256(basic_credentials->getPassword(), auth_data.getPasswordHashBinary(), auth_data.getSalt()); - - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - return checkPasswordDoubleSHA1(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::LDAP: - return external_authenticators.checkLDAPCredentials(auth_data.getLDAPServerName(), *basic_credentials); - - case AuthenticationType::KERBEROS: - throw Authentication::Require(auth_data.getKerberosRealm()); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::JWT: - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); - - case AuthenticationType::BCRYPT_PASSWORD: - return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::HTTP: - switch (auth_data.getHTTPAuthenticationScheme()) - { - case HTTPAuthenticationScheme::BASIC: - return external_authenticators.checkHTTPBasicCredentials( - auth_data.getHTTPAuthenticationServerName(), *basic_credentials, settings); - } - - case AuthenticationType::MAX: - break; - } + return checkBasicAuthentication(basic_credentials, authentication_method, external_authenticators, settings); } if (const auto * ssl_certificate_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) - { - case AuthenticationType::NO_PASSWORD: - case AuthenticationType::PLAINTEXT_PASSWORD: - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::JWT: - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); - - case AuthenticationType::KERBEROS: - throw Authentication::Require(auth_data.getKerberosRealm()); - - case AuthenticationType::SSL_CERTIFICATE: - for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN}) - { - for (const auto & subject : auth_data.getSSLCertificateSubjects().at(type)) - { - if (ssl_certificate_credentials->getSSLCertificateSubjects().at(type).contains(subject)) - return true; - } - } - return false; - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::MAX: - break; - } + return checkSSLCertificateAuthentication(ssl_certificate_credentials, authentication_method); } #if USE_SSH if (const auto * ssh_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) - { - case AuthenticationType::NO_PASSWORD: - case AuthenticationType::PLAINTEXT_PASSWORD: - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::JWT: - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud"); - - case AuthenticationType::KERBEROS: - throw Authentication::Require(auth_data.getKerberosRealm()); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: - return checkSshSignature(auth_data.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal()); - case AuthenticationType::MAX: - break; - } + return checkSshAuthentication(ssh_credentials, authentication_method); } #endif if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "areCredentialsValid(): authentication type {} not supported", toString(auth_data.getType())); + return false; } } diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index ffc497cc442..e895001304d 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -24,7 +24,7 @@ struct Authentication /// returned by the authentication server static bool areCredentialsValid( const Credentials & credentials, - const AuthenticationData & auth_data, + const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings); diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index 5a35eeefe5b..97010e67c5e 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -375,7 +375,8 @@ std::shared_ptr AuthenticationData::toAST() const break; } - case AuthenticationType::NO_PASSWORD: [[fallthrough]]; + case AuthenticationType::NO_PASSWORD: + break; case AuthenticationType::MAX: throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type)); } diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index ee422f7d8ff..046c532cf5c 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -1,8 +1,6 @@ #include #include #include -#include -#include #include #include #include @@ -418,7 +416,7 @@ void DiskAccessStorage::setAllInMemory(const std::vector & ids_to_keep) @@ -507,14 +505,14 @@ std::optional> DiskAccessStorage::readNameWi } -bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { std::lock_guard lock{mutex}; - return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk = */ true); + return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, conflicting_id, /* write_on_disk = */ true); } -bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) +bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk) { const String & name = new_entity->getName(); AccessEntityType type = new_entity->getType(); @@ -533,9 +531,15 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne if (name_collision && !replace_if_exists) { if (throw_if_exists) + { throwNameCollisionCannotInsert(type, name); + } else + { + if (conflicting_id) + *conflicting_id = id_by_name; return false; + } } auto it_by_id = entries_by_id.find(id); @@ -548,7 +552,11 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne throwIDCollisionCannotInsert(id, type, name, existing_entry.type, existing_entry.name); } else + { + if (conflicting_id) + *conflicting_id = id; return false; + } } if (write_on_disk) @@ -727,25 +735,4 @@ void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Couldn't delete {}", file_path); } - -void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) -{ - if (!isRestoreAllowed()) - throwRestoreNotAllowed(); - - auto entities = restorer.getAccessEntitiesToRestore(); - if (entities.empty()) - return; - - auto create_access = restorer.getRestoreSettings().create_access; - bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); - - restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists] - { - for (const auto & [id, entity] : my_entities) - insert(id, entity, replace_if_exists, throw_if_exists); - }); -} - } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 38172b26970..40f2017dd97 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -34,14 +34,13 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void restoreFromBackup(RestorerFromBackup & restorer) override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override; - bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; + bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; @@ -55,7 +54,7 @@ private: void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS; void stopListsWritingThread(); - bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) TSA_REQUIRES(mutex); + bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk) TSA_REQUIRES(mutex); bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex); bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex); diff --git a/src/Access/IAccessEntity.cpp b/src/Access/IAccessEntity.cpp index 5dc566fe456..9afa1b73597 100644 --- a/src/Access/IAccessEntity.cpp +++ b/src/Access/IAccessEntity.cpp @@ -9,4 +9,28 @@ bool IAccessEntity::equal(const IAccessEntity & other) const return (name == other.name) && (getType() == other.getType()); } +void IAccessEntity::replaceDependencies(std::shared_ptr & entity, const std::unordered_map & old_to_new_ids) +{ + if (old_to_new_ids.empty()) + return; + + bool need_replace_dependencies = false; + auto dependencies = entity->findDependencies(); + for (const auto & dependency : dependencies) + { + if (old_to_new_ids.contains(dependency)) + { + need_replace_dependencies = true; + break; + } + } + + if (!need_replace_dependencies) + return; + + auto new_entity = entity->clone(); + new_entity->replaceDependencies(old_to_new_ids); + entity = new_entity; +} + } diff --git a/src/Access/IAccessEntity.h b/src/Access/IAccessEntity.h index 5614a172f6f..2c2df7353c5 100644 --- a/src/Access/IAccessEntity.h +++ b/src/Access/IAccessEntity.h @@ -50,7 +50,8 @@ struct IAccessEntity virtual std::vector findDependencies() const { return {}; } /// Replaces dependencies according to a specified map. - virtual void replaceDependencies(const std::unordered_map & /* old_to_new_ids */) {} + void replaceDependencies(const std::unordered_map & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); } + static void replaceDependencies(std::shared_ptr & entity, const std::unordered_map & old_to_new_ids); /// Whether this access entity should be written to a backup. virtual bool isBackupAllowed() const { return false; } @@ -66,6 +67,8 @@ protected: { return std::make_shared(typeid_cast(*this)); } + + virtual void doReplaceDependencies(const std::unordered_map & /* old_to_new_ids */) {} }; using AccessEntityPtr = std::shared_ptr; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8d4e7d3073e..29475461c45 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -14,10 +16,11 @@ #include #include #include +#include #include +#include #include - namespace DB { namespace ErrorCodes @@ -30,7 +33,6 @@ namespace ErrorCodes extern const int IP_ADDRESS_NOT_ALLOWED; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; - extern const int AUTHENTICATION_FAILED; } @@ -179,20 +181,20 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity) return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true); } -std::optional IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) +std::optional IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { auto id = generateRandomID(); - if (insert(id, entity, replace_if_exists, throw_if_exists)) + if (insert(id, entity, replace_if_exists, throw_if_exists, conflicting_id)) return id; return std::nullopt; } -bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) +bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { - return insertImpl(id, entity, replace_if_exists, throw_if_exists); + return insertImpl(id, entity, replace_if_exists, throw_if_exists, conflicting_id); } @@ -286,7 +288,7 @@ std::vector IAccessStorage::insertOrReplace(const std::vectorgetType(), entity->getName()); @@ -525,15 +527,32 @@ std::optional IAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); - auto auth_type = user->auth_data.getType(); - if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || - ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) - throwAuthenticationTypeNotAllowed(auth_type); + bool skipped_not_allowed_authentication_methods = false; - if (!areCredentialsValid(*user, credentials, external_authenticators, auth_result.settings)) - throwInvalidCredentials(); + for (const auto & auth_method : user->authentication_methods) + { + auto auth_type = auth_method.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + { + skipped_not_allowed_authentication_methods = true; + continue; + } - return auth_result; + if (areCredentialsValid(user->getName(), user->valid_until, auth_method, credentials, external_authenticators, auth_result.settings)) + { + auth_result.authentication_data = auth_method; + return auth_result; + } + } + + if (skipped_not_allowed_authentication_methods) + { + LOG_INFO(log, "Skipped the check for not allowed authentication methods," + "check allow_no_password and allow_plaintext_password settings in the server configuration"); + } + + throwInvalidCredentials(); } } @@ -543,9 +562,10 @@ std::optional IAccessStorage::authenticateImpl( return std::nullopt; } - bool IAccessStorage::areCredentialsValid( - const User & user, + const std::string & user_name, + time_t valid_until, + const AuthenticationData & authentication_method, const Credentials & credentials, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) const @@ -553,21 +573,20 @@ bool IAccessStorage::areCredentialsValid( if (!credentials.isReady()) return false; - if (credentials.getUserName() != user.getName()) + if (credentials.getUserName() != user_name) return false; - if (user.valid_until) + if (valid_until) { const time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - if (now > user.valid_until) + if (now > valid_until) return false; } - return Authentication::areCredentialsValid(credentials, user.auth_data, external_authenticators, settings); + return Authentication::areCredentialsValid(credentials, authentication_method, external_authenticators, settings); } - bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const { return user.allowed_client_hosts.contains(address); @@ -595,12 +614,51 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c } -void IAccessStorage::restoreFromBackup(RestorerFromBackup &) +void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "restoreFromBackup() is not implemented in {}", getStorageType()); + if (isReplicated() && !acquireReplicatedRestore(restorer)) + return; + + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; + + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities_to_restore = std::move(entities), replace_if_exists, throw_if_exists] mutable + { + std::unordered_map new_to_existing_ids; + for (auto & [id, entity] : entities_to_restore) + { + UUID existing_entity_id; + if (!insert(id, entity, replace_if_exists, throw_if_exists, &existing_entity_id)) + { + /// Couldn't insert `entity` because there is an existing entity with the same name. + new_to_existing_ids[id] = existing_entity_id; + } + } + + if (!new_to_existing_ids.empty()) + { + /// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed, + /// then we should correct those dependencies. + auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr + { + auto res = entity; + IAccessEntity::replaceDependencies(res, new_to_existing_ids); + return res; + }; + std::vector ids; + ids.reserve(entities_to_restore.size()); + boost::copy(entities_to_restore | boost::adaptors::map_keys, std::back_inserter(ids)); + tryUpdate(ids, update_func); + } + }); } @@ -747,14 +805,6 @@ void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address throw Exception(ErrorCodes::IP_ADDRESS_NOT_ALLOWED, "Connections from {} are not allowed", address.toString()); } -void IAccessStorage::throwAuthenticationTypeNotAllowed(AuthenticationType auth_type) -{ - throw Exception( - ErrorCodes::AUTHENTICATION_FAILED, - "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", - toString(auth_type), AuthenticationTypeInfo::get(auth_type).name); -} - void IAccessStorage::throwInvalidCredentials() { throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials"); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index e88b1601f32..a8ac75075d3 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -34,6 +35,7 @@ struct AuthResult UUID user_id; /// Session settings received from authentication server (if any) SettingsChanges settings{}; + AuthenticationData authentication_data {}; }; /// Contains entities, i.e. instances of classes derived from IAccessEntity. @@ -62,6 +64,9 @@ public: /// Returns true if this entity is readonly. virtual bool isReadOnly(const UUID &) const { return isReadOnly(); } + /// Returns true if this storage is replicated. + virtual bool isReplicated() const { return false; } + /// Starts periodic reloading and updating of entities in this storage. virtual void startPeriodicReloading() {} @@ -151,8 +156,8 @@ public: /// Inserts an entity to the storage. Returns ID of a new entry in the storage. /// Throws an exception if the specified name already exists. UUID insert(const AccessEntityPtr & entity); - std::optional insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); - bool insert(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); + std::optional insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id = nullptr); + bool insert(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id = nullptr); std::vector insert(const std::vector & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true); std::vector insert(const std::vector & multiple_entities, const std::vector & ids, bool replace_if_exists = false, bool throw_if_exists = true); @@ -216,7 +221,7 @@ protected: virtual std::vector findAllImpl(AccessEntityType type) const = 0; virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0; virtual std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const; - virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); + virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id); virtual bool removeImpl(const UUID & id, bool throw_if_not_exists); virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); virtual std::optional authenticateImpl( @@ -227,7 +232,9 @@ protected: bool allow_no_password, bool allow_plaintext_password) const; virtual bool areCredentialsValid( - const User & user, + const std::string & user_name, + time_t valid_until, + const AuthenticationData & authentication_method, const Credentials & credentials, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) const; @@ -236,6 +243,7 @@ protected: LoggerPtr getLogger() const; static String formatEntityTypeWithName(AccessEntityType type, const String & name) { return AccessEntityTypeInfo::get(type).formatEntityNameWithType(name); } static void clearConflictsInEntitiesList(std::vector> & entities, LoggerPtr log_); + virtual bool acquireReplicatedRestore(RestorerFromBackup &) const { return false; } [[noreturn]] void throwNotFound(const UUID & id) const; [[noreturn]] void throwNotFound(AccessEntityType type, const String & name) const; [[noreturn]] static void throwBadCast(const UUID & id, AccessEntityType type, const String & name, AccessEntityType required_type); @@ -248,7 +256,6 @@ protected: [[noreturn]] void throwReadonlyCannotRemove(AccessEntityType type, const String & name) const; [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address); [[noreturn]] static void throwInvalidCredentials(); - [[noreturn]] static void throwAuthenticationTypeNotAllowed(AuthenticationType auth_type); [[noreturn]] void throwBackupNotAllowed() const; [[noreturn]] void throwRestoreNotAllowed() const; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 3206b20b691..f6435b0d899 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -468,8 +468,8 @@ std::optional LDAPAccessStorage::authenticateImpl( // User does not exist, so we create one, and will add it if authentication is successful. new_user = std::make_shared(); new_user->setName(credentials.getUserName()); - new_user->auth_data = AuthenticationData(AuthenticationType::LDAP); - new_user->auth_data.setLDAPServerName(ldap_server_name); + new_user->authentication_methods.emplace_back(AuthenticationType::LDAP); + new_user->authentication_methods.back().setLDAPServerName(ldap_server_name); user = new_user; } @@ -504,7 +504,7 @@ std::optional LDAPAccessStorage::authenticateImpl( } if (id) - return AuthResult{ .user_id = *id }; + return AuthResult{ .user_id = *id, .authentication_data = AuthenticationData(AuthenticationType::LDAP) }; return std::nullopt; } diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 791030b9b12..3b5a987fc6e 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -1,7 +1,5 @@ #include #include -#include -#include #include #include #include @@ -63,14 +61,14 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not } -bool MemoryAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool MemoryAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { std::lock_guard lock{mutex}; - return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists); + return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, conflicting_id); } -bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { const String & name = new_entity->getName(); AccessEntityType type = new_entity->getType(); @@ -86,9 +84,15 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & if (name_collision && !replace_if_exists) { if (throw_if_exists) + { throwNameCollisionCannotInsert(type, name); + } else + { + if (conflicting_id) + *conflicting_id = id_by_name; return false; + } } auto it_by_id = entries_by_id.find(id); @@ -97,9 +101,15 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & { const auto & existing_entry = it_by_id->second; if (throw_if_exists) + { throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); + } else + { + if (conflicting_id) + *conflicting_id = id; return false; + } } /// Remove collisions if necessary. @@ -270,28 +280,7 @@ void MemoryAccessStorage::setAll(const std::vector findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; - bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; + bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); + bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id); bool removeNoLock(const UUID & id, bool throw_if_not_exists); bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index fda6601e4c6..f1da8359d48 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -353,7 +353,7 @@ void MultipleAccessStorage::reload(ReloadMode reload_mode) } -bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) +bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { std::shared_ptr storage_for_insertion; @@ -376,7 +376,7 @@ bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & getStorageName()); } - if (storage_for_insertion->insert(id, entity, replace_if_exists, throw_if_exists)) + if (storage_for_insertion->insert(id, entity, replace_if_exists, throw_if_exists, conflicting_id)) { std::lock_guard lock{mutex}; ids_cache.set(id, storage_for_insertion); diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index e1543c59b67..352cc7f7457 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -67,7 +67,7 @@ protected: std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; std::optional> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override; - bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; + bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override; diff --git a/src/Access/Quota.cpp b/src/Access/Quota.cpp index 87b15e722c3..ead5f77ce57 100644 --- a/src/Access/Quota.cpp +++ b/src/Access/Quota.cpp @@ -24,7 +24,7 @@ std::vector Quota::findDependencies() const return to_roles.findDependencies(); } -void Quota::replaceDependencies(const std::unordered_map & old_to_new_ids) +void Quota::doReplaceDependencies(const std::unordered_map & old_to_new_ids) { to_roles.replaceDependencies(old_to_new_ids); } diff --git a/src/Access/Quota.h b/src/Access/Quota.h index eb9edb14fb0..69ec2eb53a5 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -47,7 +47,7 @@ struct Quota : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; bool isBackupAllowed() const override { return true; } }; diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index ed114327041..9039a3b98b7 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -5,10 +5,9 @@ #include #include #include -#include -#include #include #include +#include #include #include #include @@ -120,7 +119,7 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function) } } -bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) +bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) { const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType()); const String & name = new_entity->getName(); @@ -128,7 +127,7 @@ bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr auto zookeeper = getZooKeeper(); bool ok = false; - retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists); }); + retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists, conflicting_id); }); if (!ok) return false; @@ -143,7 +142,8 @@ bool ReplicatedAccessStorage::insertZooKeeper( const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, - bool throw_if_exists) + bool throw_if_exists, + UUID * conflicting_id) { const String & name = new_entity->getName(); const AccessEntityType type = new_entity->getType(); @@ -167,27 +167,52 @@ bool ReplicatedAccessStorage::insertZooKeeper( if (res == Coordination::Error::ZNODEEXISTS) { - if (!throw_if_exists && !replace_if_exists) - return false; /// Couldn't insert a new entity. - - if (throw_if_exists) + if (!replace_if_exists) { if (responses[0]->error == Coordination::Error::ZNODEEXISTS) { - /// To fail with a nice error message, we need info about what already exists. - /// This itself could fail if the conflicting uuid disappears in the meantime. - /// If that happens, then we'll just retry from the start. - String existing_entity_definition = zookeeper->get(entity_path); + /// Couldn't insert the new entity because there is an existing entity with such UUID. + if (throw_if_exists) + { + /// To fail with a nice error message, we need info about what already exists. + /// This itself can fail if the conflicting uuid disappears in the meantime. + /// If that happens, then retryOnZooKeeperUserError() will just retry the operation from the start. + String existing_entity_definition = zookeeper->get(entity_path); - AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); - AccessEntityType existing_type = existing_entity->getType(); - String existing_name = existing_entity->getName(); - throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); + AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); + AccessEntityType existing_type = existing_entity->getType(); + String existing_name = existing_entity->getName(); + throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); + } + else + { + if (conflicting_id) + *conflicting_id = id; + return false; + } + } + else if (responses[1]->error == Coordination::Error::ZNODEEXISTS) + { + /// Couldn't insert the new entity because there is an existing entity with the same name. + if (throw_if_exists) + { + throwNameCollisionCannotInsert(type, name); + } + else + { + if (conflicting_id) + { + /// Get UUID of the existing entry with the same name. + /// This itself can fail if the conflicting name disappears in the meantime. + /// If that happens, then retryOnZooKeeperUserError() will just retry the operation from the start. + *conflicting_id = parseUUID(zookeeper->get(name_path)); + } + return false; + } } else { - /// Couldn't insert the new entity because there is an existing entity with such name. - throwNameCollisionCannotInsert(type, name); + zkutil::KeeperMultiException::check(res, ops, responses); } } @@ -693,28 +718,10 @@ void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_col } -void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) +bool ReplicatedAccessStorage::acquireReplicatedRestore(RestorerFromBackup & restorer) const { - if (!isRestoreAllowed()) - throwRestoreNotAllowed(); - auto restore_coordination = restorer.getRestoreCoordination(); - if (!restore_coordination->acquireReplicatedAccessStorage(zookeeper_path)) - return; - - auto entities = restorer.getAccessEntitiesToRestore(); - if (entities.empty()) - return; - - auto create_access = restorer.getRestoreSettings().create_access; - bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); - - restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists] - { - for (const auto & [id, entity] : my_entities) - insert(id, entity, replace_if_exists, throw_if_exists); - }); + return restore_coordination->acquireReplicatedAccessStorage(zookeeper_path); } } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index f8518226997..528dbb31c24 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -26,6 +26,7 @@ public: void shutdown() override; const char * getStorageType() const override { return STORAGE_TYPE; } + bool isReplicated() const override { return true; } void startPeriodicReloading() override { startWatchingThread(); } void stopPeriodicReloading() override { stopWatchingThread(); } @@ -35,7 +36,6 @@ public: bool isBackupAllowed() const override { return backup_allowed; } void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; - void restoreFromBackup(RestorerFromBackup & restorer) override; private: String zookeeper_path; @@ -48,11 +48,11 @@ private: std::unique_ptr watching_thread; std::shared_ptr> watched_queue; - bool insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) override; + bool insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); + bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id); bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists); bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); @@ -80,6 +80,7 @@ private: std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; + bool acquireReplicatedRestore(RestorerFromBackup & restorer) const override; mutable std::mutex mutex; MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex); diff --git a/src/Access/Role.cpp b/src/Access/Role.cpp index 089488e7aba..f6250594103 100644 --- a/src/Access/Role.cpp +++ b/src/Access/Role.cpp @@ -21,7 +21,7 @@ std::vector Role::findDependencies() const return res; } -void Role::replaceDependencies(const std::unordered_map & old_to_new_ids) +void Role::doReplaceDependencies(const std::unordered_map & old_to_new_ids) { granted_roles.replaceDependencies(old_to_new_ids); settings.replaceDependencies(old_to_new_ids); diff --git a/src/Access/Role.h b/src/Access/Role.h index b2f879dc357..c7f98585a6c 100644 --- a/src/Access/Role.h +++ b/src/Access/Role.h @@ -21,7 +21,7 @@ struct Role : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; bool isBackupAllowed() const override { return settings.isBackupAllowed(); } }; diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index d25b9e259b1..8724d0f513c 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -63,7 +63,7 @@ std::vector RowPolicy::findDependencies() const return to_roles.findDependencies(); } -void RowPolicy::replaceDependencies(const std::unordered_map & old_to_new_ids) +void RowPolicy::doReplaceDependencies(const std::unordered_map & old_to_new_ids) { to_roles.replaceDependencies(old_to_new_ids); } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 9c190458620..5cfe85c186a 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -50,7 +50,7 @@ struct RowPolicy : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; bool isBackupAllowed() const override { return true; } /// Which roles or users should use this row policy. diff --git a/src/Access/SettingsProfile.cpp b/src/Access/SettingsProfile.cpp index 48aa48040ab..632bd97fbf5 100644 --- a/src/Access/SettingsProfile.cpp +++ b/src/Access/SettingsProfile.cpp @@ -21,7 +21,7 @@ std::vector SettingsProfile::findDependencies() const return res; } -void SettingsProfile::replaceDependencies(const std::unordered_map & old_to_new_ids) +void SettingsProfile::doReplaceDependencies(const std::unordered_map & old_to_new_ids) { elements.replaceDependencies(old_to_new_ids); to_roles.replaceDependencies(old_to_new_ids); diff --git a/src/Access/SettingsProfile.h b/src/Access/SettingsProfile.h index f85630d324d..6bcaf6fef30 100644 --- a/src/Access/SettingsProfile.h +++ b/src/Access/SettingsProfile.h @@ -22,7 +22,7 @@ struct SettingsProfile : public IAccessEntity AccessEntityType getType() const override { return TYPE; } std::vector findDependencies() const override; - void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; bool isBackupAllowed() const override { return elements.isBackupAllowed(); } }; diff --git a/src/Access/User.cpp b/src/Access/User.cpp index c02c598ee40..2052527f4ae 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -16,7 +16,8 @@ bool User::equal(const IAccessEntity & other) const if (!IAccessEntity::equal(other)) return false; const auto & other_user = typeid_cast(other); - return (auth_data == other_user.auth_data) && (allowed_client_hosts == other_user.allowed_client_hosts) + return (authentication_methods == other_user.authentication_methods) + && (allowed_client_hosts == other_user.allowed_client_hosts) && (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles) && (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database) && (valid_until == other_user.valid_until); @@ -48,7 +49,7 @@ std::vector User::findDependencies() const return res; } -void User::replaceDependencies(const std::unordered_map & old_to_new_ids) +void User::doReplaceDependencies(const std::unordered_map & old_to_new_ids) { default_roles.replaceDependencies(old_to_new_ids); granted_roles.replaceDependencies(old_to_new_ids); diff --git a/src/Access/User.h b/src/Access/User.h index e4ab654dafd..7f91c1e3756 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -15,7 +15,7 @@ namespace DB */ struct User : public IAccessEntity { - AuthenticationData auth_data; + std::vector authentication_methods; AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; AccessRights access; GrantedRoles granted_roles; @@ -32,7 +32,7 @@ struct User : public IAccessEntity void setName(const String & name_) override; std::vector findDependencies() const override; - void replaceDependencies(const std::unordered_map & old_to_new_ids) override; + void doReplaceDependencies(const std::unordered_map & old_to_new_ids) override; bool isBackupAllowed() const override { return settings.isBackupAllowed(); } }; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index a030ae96cbb..8f6761766c8 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -155,18 +155,18 @@ namespace if (has_password_plaintext) { - user->auth_data = AuthenticationData{AuthenticationType::PLAINTEXT_PASSWORD}; - user->auth_data.setPassword(config.getString(user_config + ".password")); + user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD); + user->authentication_methods.back().setPassword(config.getString(user_config + ".password")); } else if (has_password_sha256_hex) { - user->auth_data = AuthenticationData{AuthenticationType::SHA256_PASSWORD}; - user->auth_data.setPasswordHashHex(config.getString(user_config + ".password_sha256_hex")); + user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD); + user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex")); } else if (has_password_double_sha1_hex) { - user->auth_data = AuthenticationData{AuthenticationType::DOUBLE_SHA1_PASSWORD}; - user->auth_data.setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex")); + user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD); + user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex")); } else if (has_ldap) { @@ -178,19 +178,19 @@ namespace if (ldap_server_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "LDAP server name cannot be empty for user {}.", user_name); - user->auth_data = AuthenticationData{AuthenticationType::LDAP}; - user->auth_data.setLDAPServerName(ldap_server_name); + user->authentication_methods.emplace_back(AuthenticationType::LDAP); + user->authentication_methods.back().setLDAPServerName(ldap_server_name); } else if (has_kerberos) { const auto realm = config.getString(user_config + ".kerberos.realm", ""); - user->auth_data = AuthenticationData{AuthenticationType::KERBEROS}; - user->auth_data.setKerberosRealm(realm); + user->authentication_methods.emplace_back(AuthenticationType::KERBEROS); + user->authentication_methods.back().setKerberosRealm(realm); } else if (has_certificates) { - user->auth_data = AuthenticationData{AuthenticationType::SSL_CERTIFICATE}; + user->authentication_methods.emplace_back(AuthenticationType::SSL_CERTIFICATE); /// Fill list of allowed certificates. Poco::Util::AbstractConfiguration::Keys keys; @@ -200,14 +200,14 @@ namespace if (key.starts_with("common_name")) { String value = config.getString(certificates_config + "." + key); - user->auth_data.addSSLCertificateSubject(SSLCertificateSubjects::Type::CN, std::move(value)); + user->authentication_methods.back().addSSLCertificateSubject(SSLCertificateSubjects::Type::CN, std::move(value)); } else if (key.starts_with("subject_alt_name")) { String value = config.getString(certificates_config + "." + key); if (value.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected ssl_certificates.subject_alt_name to not be empty"); - user->auth_data.addSSLCertificateSubject(SSLCertificateSubjects::Type::SAN, std::move(value)); + user->authentication_methods.back().addSSLCertificateSubject(SSLCertificateSubjects::Type::SAN, std::move(value)); } else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown certificate pattern type: {}", key); @@ -216,7 +216,7 @@ namespace else if (has_ssh_keys) { #if USE_SSH - user->auth_data = AuthenticationData{AuthenticationType::SSH_KEY}; + user->authentication_methods.emplace_back(AuthenticationType::SSH_KEY); Poco::Util::AbstractConfiguration::Keys entries; config.keys(ssh_keys_config, entries); @@ -253,26 +253,33 @@ namespace else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown ssh_key entry pattern type: {}", entry); } - user->auth_data.setSSHKeys(std::move(keys)); + user->authentication_methods.back().setSSHKeys(std::move(keys)); #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } else if (has_http_auth) { - user->auth_data = AuthenticationData{AuthenticationType::HTTP}; - user->auth_data.setHTTPAuthenticationServerName(config.getString(http_auth_config + ".server")); + user->authentication_methods.emplace_back(AuthenticationType::HTTP); + user->authentication_methods.back().setHTTPAuthenticationServerName(config.getString(http_auth_config + ".server")); auto scheme = config.getString(http_auth_config + ".scheme"); - user->auth_data.setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); + user->authentication_methods.back().setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); + } + else + { + user->authentication_methods.emplace_back(); } - auto auth_type = user->auth_data.getType(); - if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || - ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + for (const auto & authentication_method : user->authentication_methods) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", - toString(auth_type), AuthenticationTypeInfo::get(auth_type).name); + auto auth_type = authentication_method.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", + toString(auth_type), AuthenticationTypeInfo::get(auth_type).name); + } } const auto profile_name_config = user_config + ".profile"; diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h index 439ddffe5e5..c598adfd98e 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h @@ -3,370 +3,89 @@ #include #include #include -#include #include -#include -#include -#include - -#include namespace DB { +class FunctionTreeNode : public AbstractFunction +{ +public: + class ArgumentTreeNode : public Argument + { + public: + explicit ArgumentTreeNode(const IQueryTreeNode * argument_) : argument(argument_) {} + std::unique_ptr getFunction() const override + { + if (const auto * f = argument->as()) + return std::make_unique(*f); + return nullptr; + } + bool isIdentifier() const override { return argument->as(); } + bool tryGetString(String * res, bool allow_identifier) const override + { + if (const auto * literal = argument->as()) + { + if (literal->getValue().getType() != Field::Types::String) + return false; + if (res) + *res = literal->getValue().safeGet(); + return true; + } + + if (allow_identifier) + { + if (const auto * id = argument->as()) + { + if (res) + *res = id->getIdentifier().getFullName(); + return true; + } + } + + return false; + } + private: + const IQueryTreeNode * argument = nullptr; + }; + + class ArgumentsTreeNode : public Arguments + { + public: + explicit ArgumentsTreeNode(const QueryTreeNodes * arguments_) : arguments(arguments_) {} + size_t size() const override { return arguments ? arguments->size() : 0; } + std::unique_ptr at(size_t n) const override { return std::make_unique(arguments->at(n).get()); } + private: + const QueryTreeNodes * arguments = nullptr; + }; + + explicit FunctionTreeNode(const FunctionNode & function_) : function(&function_) + { + if (const auto & nodes = function->getArguments().getNodes(); !nodes.empty()) + arguments = std::make_unique(&nodes); + } + String name() const override { return function->getFunctionName(); } +private: + const FunctionNode * function = nullptr; +}; + /// Finds arguments of a specified function which should not be displayed for most users for security reasons. /// That involves passwords and secret keys. -class FunctionSecretArgumentsFinderTreeNode +class FunctionSecretArgumentsFinderTreeNode : public FunctionSecretArgumentsFinder { public: - explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_) : function(function_), arguments(function.getArguments()) + explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_) + : FunctionSecretArgumentsFinder(std::make_unique(function_)) { - if (arguments.getNodes().empty()) + if (!function->hasArguments()) return; - findFunctionSecretArguments(); + findOrdinaryFunctionSecretArguments(); } - struct Result - { - /// Result constructed by default means no arguments will be hidden. - size_t start = static_cast(-1); - size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`). - /// In all known cases secret arguments are consecutive - bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments. - /// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))` - std::vector nested_maps; - - bool hasSecrets() const - { - return count != 0 || !nested_maps.empty(); - } - }; - FunctionSecretArgumentsFinder::Result getResult() const { return result; } - -private: - const FunctionNode & function; - const ListNode & arguments; - FunctionSecretArgumentsFinder::Result result; - - void markSecretArgument(size_t index, bool argument_is_named = false) - { - if (index >= arguments.getNodes().size()) - return; - if (!result.count) - { - result.start = index; - result.are_named = argument_is_named; - } - chassert(index >= result.start); /// We always check arguments consecutively - result.count = index + 1 - result.start; - if (!argument_is_named) - result.are_named = false; - } - - void findFunctionSecretArguments() - { - const auto & name = function.getFunctionName(); - - if ((name == "mysql") || (name == "postgresql") || (name == "mongodb")) - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - /// postgresql('host:port', 'database', 'table', 'user', 'password', ...) - /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); - } - else if ((name == "s3") || (name == "cosn") || (name == "oss") || - (name == "deltaLake") || (name == "hudi") || (name == "iceberg")) - { - /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ false); - } - else if (name == "s3Cluster") - { - /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ true); - } - else if ((name == "remote") || (name == "remoteSecure")) - { - /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...) - findRemoteFunctionSecretArguments(); - } - else if ((name == "encrypt") || (name == "decrypt") || - (name == "aes_encrypt_mysql") || (name == "aes_decrypt_mysql") || - (name == "tryDecrypt")) - { - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) - findEncryptionFunctionSecretArguments(); - } - else if (name == "url") - { - findURLSecretArguments(); - } - } - - void findMySQLFunctionSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// mysql(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - } - else - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - markSecretArgument(4); - } - } - - /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should - /// always be at the end). Marks "headers" as secret, if found. - size_t excludeS3OrURLNestedMaps() - { - const auto & nodes = arguments.getNodes(); - size_t count = nodes.size(); - while (count > 0) - { - const FunctionNode * f = nodes.at(count - 1)->as(); - if (!f) - break; - if (f->getFunctionName() == "headers") - result.nested_maps.push_back(f->getFunctionName()); - else if (f->getFunctionName() != "extra_credentials") - break; - count -= 1; - } - return count; - } - - void findS3FunctionSecretArguments(bool is_cluster_function) - { - /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. - size_t url_arg_idx = is_cluster_function ? 1 : 0; - - if (!is_cluster_function && isNamedCollectionName(0)) - { - /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...) - findSecretNamedArgument("secret_access_key", 1); - return; - } - - /// We should check other arguments first because we don't need to do any replacement in case of - /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - size_t count = excludeS3OrURLNestedMaps(); - if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4)) - { - String second_arg; - if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg)) - { - if (boost::iequals(second_arg, "NOSIGN")) - return; /// The argument after 'url' is "NOSIGN". - - if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) - return; /// The argument after 'url' is a format: s3('url', 'format', ...) - } - } - - /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: - /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - if (url_arg_idx + 2 < count) - markSecretArgument(url_arg_idx + 2); - } - - void findURLSecretArguments() - { - if (!isNamedCollectionName(0)) - excludeS3OrURLNestedMaps(); - } - - bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const - { - if (arg_idx >= arguments.getNodes().size()) - return false; - - return tryGetStringFromArgument(arguments.getNodes()[arg_idx], res, allow_identifier); - } - - static bool tryGetStringFromArgument(const QueryTreeNodePtr argument, String * res, bool allow_identifier = true) - { - if (const auto * literal = argument->as()) - { - if (literal->getValue().getType() != Field::Types::String) - return false; - if (res) - *res = literal->getValue().safeGet(); - return true; - } - - if (allow_identifier) - { - if (const auto * id = argument->as()) - { - if (res) - *res = id->getIdentifier().getFullName(); - return true; - } - } - - return false; - } - - void findRemoteFunctionSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// remote(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - return; - } - - /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: - /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key]) - /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key]) - /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key]) - - /// But we should check the number of arguments first because we don't need to do any replacements in case of - /// remote('addresses_expr', db.table) - if (arguments.getNodes().size() < 3) - return; - - size_t arg_num = 1; - - /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'. - const auto * table_function = arguments.getNodes()[arg_num]->as(); - if (table_function && KnownTableFunctionNames::instance().exists(table_function->getFunctionName())) - { - ++arg_num; - } - else - { - std::optional database; - std::optional qualified_table_name; - if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name)) - { - /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'. - /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user' - /// before the argument 'password'. So it's safer to wipe two arguments just in case. - /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string - /// before wiping it (because the `password` argument is always a literal string). - if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false)) - { - /// Wipe either `password` or `user`. - markSecretArgument(arg_num + 2); - } - if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false)) - { - /// Wipe either `password` or `sharding_key`. - markSecretArgument(arg_num + 3); - } - return; - } - - /// Skip the current argument (which is either a database name or a qualified table name). - ++arg_num; - if (database) - { - /// Skip the 'table' argument if the previous argument was a database name. - ++arg_num; - } - } - - /// Skip username. - ++arg_num; - - /// Do our replacement: - /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...) - /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string - /// before wiping it (because the `password` argument is always a literal string). - bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false); - if (can_be_password) - markSecretArgument(arg_num); - } - - /// Tries to get either a database name or a qualified table name from an argument. - /// Empty string is also allowed (it means the default database). - /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password. - bool tryGetDatabaseNameOrQualifiedTableName( - size_t arg_idx, - std::optional & res_database, - std::optional & res_qualified_table_name) const - { - res_database.reset(); - res_qualified_table_name.reset(); - - String str; - if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true)) - return false; - - if (str.empty()) - { - res_database = ""; - return true; - } - - auto qualified_table_name = QualifiedTableName::tryParseFromString(str); - if (!qualified_table_name) - return false; - - if (qualified_table_name->database.empty()) - res_database = std::move(qualified_table_name->table); - else - res_qualified_table_name = std::move(qualified_table_name); - return true; - } - - void findEncryptionFunctionSecretArguments() - { - if (arguments.getNodes().empty()) - return; - - /// We replace all arguments after 'mode' with '[HIDDEN]': - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]') - result.start = 1; - result.count = arguments.getNodes().size() - 1; - } - - - /// Whether a specified argument can be the name of a named collection? - bool isNamedCollectionName(size_t arg_idx) const - { - if (arguments.getNodes().size() <= arg_idx) - return false; - - const auto * identifier = arguments.getNodes()[arg_idx]->as(); - return identifier != nullptr; - } - - /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified. - void findSecretNamedArgument(const std::string_view & key, size_t start = 0) - { - for (size_t i = start; i < arguments.getNodes().size(); ++i) - { - const auto & argument = arguments.getNodes()[i]; - const auto * equals_func = argument->as(); - if (!equals_func || (equals_func->getFunctionName() != "equals")) - continue; - - const auto * expr_list = equals_func->getArguments().as(); - if (!expr_list) - continue; - - const auto & equal_args = expr_list->getNodes(); - if (equal_args.size() != 2) - continue; - - String found_key; - if (!tryGetStringFromArgument(equal_args[0], &found_key)) - continue; - - if (found_key == key) - markSecretArgument(i, /* argument_is_named= */ true); - } - } }; } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 717a9bbe95a..5c86d911f96 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1875,11 +1875,11 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (const auto * create_user_query = parsed_query->as()) { - if (!create_user_query->attach && create_user_query->auth_data) + if (!create_user_query->attach && !create_user_query->authentication_methods.empty()) { - if (const auto * auth_data = create_user_query->auth_data->as()) + for (const auto & authentication_method : create_user_query->authentication_methods) { - auto password = auth_data->getPassword(); + auto password = authentication_method->getPassword(); if (password) client_context->getAccessControl().checkPasswordComplexityRules(*password); diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index da6e5baa3ad..8a1c7d3988a 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -455,6 +455,9 @@ void Connection::sendAddendum() writeStringBinary(proto_recv_chunked, *out); } + if (server_revision >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + writeVarUInt(DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION, *out); + out->next(); } @@ -525,6 +528,8 @@ void Connection::receiveHello(const Poco::Timespan & handshake_timeout) readVarUInt(server_version_major, *in); readVarUInt(server_version_minor, *in); readVarUInt(server_revision, *in); + if (server_revision >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + readVarUInt(server_parallel_replicas_protocol_version, *in); if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) readStringBinary(server_timezone, *in); if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) @@ -959,7 +964,7 @@ void Connection::sendReadTaskResponse(const String & response) void Connection::sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) { writeVarUInt(Protocol::Client::MergeTreeReadTaskResponse, *out); - response.serialize(*out); + response.serialize(*out, server_parallel_replicas_protocol_version); out->finishChunk(); out->next(); } @@ -1413,7 +1418,7 @@ ParallelReadRequest Connection::receiveParallelReadRequest() const InitialAllRangesAnnouncement Connection::receiveInitialParallelReadAnnouncement() const { - return InitialAllRangesAnnouncement::deserialize(*in); + return InitialAllRangesAnnouncement::deserialize(*in, server_parallel_replicas_protocol_version); } diff --git a/src/Client/Connection.h b/src/Client/Connection.h index ed84bc51318..e09d913f1ba 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -210,6 +210,7 @@ private: UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; UInt64 server_revision = 0; + UInt64 server_parallel_replicas_protocol_version = 0; String server_timezone; String server_display_name; diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 4bf2b0704f1..658eaedbda1 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -178,6 +178,9 @@ M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \ M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \ M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \ + M(BuildVectorSimilarityIndexThreads, "Number of threads in the build vector similarity index thread pool.") \ + M(BuildVectorSimilarityIndexThreadsActive, "Number of threads in the build vector similarity index thread pool running a task.") \ + M(BuildVectorSimilarityIndexThreadsScheduled, "Number of queued or active jobs in the build vector similarity index thread pool.") \ \ M(DiskPlainRewritableAzureDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \ M(DiskPlainRewritableLocalDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for LocalObjectStorage.") \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 1055b3d34db..09a5375191b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -609,6 +609,7 @@ M(728, UNEXPECTED_DATA_TYPE) \ M(729, ILLEGAL_TIME_SERIES_TAGS) \ M(730, REFRESH_FAILED) \ + M(731, QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index b2fcbc77c56..b36d438d1e3 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -63,6 +63,8 @@ static struct InitFiu REGULAR(keepermap_fail_drop_data) \ REGULAR(lazy_pipe_fds_fail_close) \ PAUSEABLE(infinite_sleep) \ + PAUSEABLE(stop_moving_part_before_swap_with_active) \ + REGULAR(slowdown_index_analysis) \ namespace FailPoints diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 467dfe60cd7..7c28b193d13 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -376,6 +376,7 @@ The server successfully detected this situation and will download merged part fr M(ParallelReplicasReadAssignedMarks, "Sum across all replicas of how many of scheduled marks were assigned by consistent hash") \ M(ParallelReplicasReadUnassignedMarks, "Sum across all replicas of how many unassigned marks were scheduled") \ M(ParallelReplicasReadAssignedForStealingMarks, "Sum across all replicas of how many of scheduled marks were assigned for stealing by consistent hash") \ + M(ParallelReplicasReadMarks, "How many marks were read by the given replica") \ \ M(ParallelReplicasStealingByHashMicroseconds, "Time spent collecting segments meant for stealing by hash") \ M(ParallelReplicasProcessingPartsMicroseconds, "Time spent processing data parts") \ @@ -529,6 +530,7 @@ The server successfully detected this situation and will download merged part fr M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \ M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \ M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache") \ + M(CachedReadBufferPredownloadedBytes, "Bytes read from filesystem cache source. Cache segments are read from left to right as a whole, it might be that we need to predownload some part of the segment irrelevant for the current task just to get to the needed data") \ M(CachedReadBufferCacheWriteBytes, "Bytes written from source (remote fs, etc) to filesystem cache") \ M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into filesystem cache") \ M(CachedReadBufferCreateBufferMicroseconds, "Prepare buffer time") \ diff --git a/src/Common/ZooKeeper/IKeeper.cpp b/src/Common/ZooKeeper/IKeeper.cpp index 7cca262baca..34c9d94fca5 100644 --- a/src/Common/ZooKeeper/IKeeper.cpp +++ b/src/Common/ZooKeeper/IKeeper.cpp @@ -171,6 +171,7 @@ bool isUserError(Error zk_return_code) void CreateRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void RemoveRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } +void RemoveRecursiveRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void ExistsRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void GetRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void SetRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index ce7489a33e5..a0d6ae54f56 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -248,6 +248,23 @@ struct RemoveResponse : virtual Response { }; +struct RemoveRecursiveRequest : virtual Request +{ + String path; + + /// strict limit for number of deleted nodes + uint32_t remove_nodes_limit = 1; + + void addRootPath(const String & root_path) override; + String getPath() const override { return path; } + + size_t bytesSize() const override { return path.size() + sizeof(remove_nodes_limit); } +}; + +struct RemoveRecursiveResponse : virtual Response +{ +}; + struct ExistsRequest : virtual Request { String path; @@ -430,6 +447,7 @@ struct ErrorResponse : virtual Response using CreateCallback = std::function; using RemoveCallback = std::function; +using RemoveRecursiveCallback = std::function; using ExistsCallback = std::function; using GetCallback = std::function; using SetCallback = std::function; @@ -587,6 +605,11 @@ public: int32_t version, RemoveCallback callback) = 0; + virtual void removeRecursive( + const String & path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) = 0; + virtual void exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 16ea412eb77..2fbe9110b6b 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -90,6 +90,36 @@ struct TestKeeperRemoveRequest final : RemoveRequest, TestKeeperRequest } }; +struct TestKeeperRemoveRecursiveRequest final : RemoveRecursiveRequest, TestKeeperRequest +{ + TestKeeperRemoveRecursiveRequest() = default; + explicit TestKeeperRemoveRecursiveRequest(const RemoveRecursiveRequest & base) : RemoveRecursiveRequest(base) {} + ResponsePtr createResponse() const override; + std::pair process(TestKeeper::Container & container, int64_t zxid) const override; + + void processWatches(TestKeeper::Watches & node_watches, TestKeeper::Watches & list_watches) const override + { + std::vector> deleted; + + auto add_deleted_watches = [&](TestKeeper::Watches & w) + { + for (const auto & [watch_path, _] : w) + if (watch_path.starts_with(path)) + deleted.emplace_back(watch_path, std::count(watch_path.begin(), watch_path.end(), '/')); + }; + + add_deleted_watches(node_watches); + add_deleted_watches(list_watches); + std::sort(deleted.begin(), deleted.end(), [](const auto & lhs, const auto & rhs) + { + return lhs.second < rhs.second; + }); + + for (const auto & [watch_path, _] : deleted) + processWatchesImpl(watch_path, node_watches, list_watches); + } +}; + struct TestKeeperExistsRequest final : ExistsRequest, TestKeeperRequest { ResponsePtr createResponse() const override; @@ -175,6 +205,10 @@ struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest { requests.push_back(std::make_shared(*concrete_request_remove)); } + else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) + { + requests.push_back(std::make_shared(*concrete_request_remove_recursive)); + } else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_set)); @@ -313,6 +347,62 @@ std::pair TestKeeperRemoveRequest::process(TestKeeper::Contai return { std::make_shared(response), undo }; } +std::pair TestKeeperRemoveRecursiveRequest::process(TestKeeper::Container & container, int64_t zxid) const +{ + RemoveRecursiveResponse response; + response.zxid = zxid; + Undo undo; + + auto root_it = container.find(path); + if (root_it == container.end()) + { + response.error = Error::ZNONODE; + return { std::make_shared(response), undo }; + } + + std::vector> children; + + for (auto it = std::next(root_it); it != container.end(); ++it) + { + const auto & [child_path, child_node] = *it; + + if (child_path.starts_with(path)) + children.emplace_back(child_path, child_node); + else + break; + } + + if (children.size() > remove_nodes_limit) + { + response.error = Error::ZNOTEMPTY; + return { std::make_shared(response), undo }; + } + + auto & parent = container.at(parentPath(path)); + --parent.stat.numChildren; + ++parent.stat.cversion; + + for (const auto & [child_path, child_node] : children) + { + auto child_it = container.find(child_path); + chassert(child_it != container.end()); + container.erase(child_it); + } + + response.error = Error::ZOK; + undo = [&container, dead = std::move(children), root_path = path]() + { + for (auto && [child_path, child_node] : dead) + container.emplace(child_path, child_node); + + auto & undo_parent = container.at(parentPath(root_path)); + ++undo_parent.stat.numChildren; + --undo_parent.stat.cversion; + }; + + return { std::make_shared(response), undo }; +} + std::pair TestKeeperExistsRequest::process(TestKeeper::Container & container, int64_t zxid) const { ExistsResponse response; @@ -530,6 +620,7 @@ std::pair TestKeeperMultiRequest::process(TestKeeper::Contain ResponsePtr TestKeeperCreateRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperRemoveRequest::createResponse() const { return std::make_shared(); } +ResponsePtr TestKeeperRemoveRecursiveRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperExistsRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperGetRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperSetRequest::createResponse() const { return std::make_shared(); } @@ -771,6 +862,21 @@ void TestKeeper::remove( pushRequest(std::move(request_info)); } +void TestKeeper::removeRecursive( + const String & path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) +{ + TestKeeperRemoveRecursiveRequest request; + request.path = path; + request.remove_nodes_limit = remove_nodes_limit; + + RequestInfo request_info; + request_info.request = std::make_shared(std::move(request)); + request_info.callback = [callback](const Response & response) { callback(dynamic_cast(response)); }; + pushRequest(std::move(request_info)); +} + void TestKeeper::exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index 562c313ac0e..c32f0064dec 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -58,6 +58,11 @@ public: int32_t version, RemoveCallback callback) override; + void removeRecursive( + const String & path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) override; + void exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/Types.h b/src/Common/ZooKeeper/Types.h index d2876adaabc..4a163c15838 100644 --- a/src/Common/ZooKeeper/Types.h +++ b/src/Common/ZooKeeper/Types.h @@ -31,6 +31,7 @@ using AsyncResponses = std::vector>>; Coordination::RequestPtr makeCreateRequest(const std::string & path, const std::string & data, int create_mode, bool ignore_if_exists = false); Coordination::RequestPtr makeRemoveRequest(const std::string & path, int version); +Coordination::RequestPtr makeRemoveRecursiveRequest(const std::string & path, uint32_t remove_nodes_limit); Coordination::RequestPtr makeSetRequest(const std::string & path, const std::string & data, int version); Coordination::RequestPtr makeCheckRequest(const std::string & path, int version); diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 1a9ed4f1ee7..ae60520affb 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -979,18 +979,47 @@ bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probab return removed_as_expected; } -void ZooKeeper::removeRecursive(const std::string & path) +void ZooKeeper::removeRecursive(const std::string & path, uint32_t remove_nodes_limit) { - removeChildrenRecursive(path); - remove(path); + if (!isFeatureEnabled(DB::KeeperFeatureFlag::REMOVE_RECURSIVE)) + { + removeChildrenRecursive(path); + remove(path); + return; + } + + check(tryRemoveRecursive(path, remove_nodes_limit), path); } -void ZooKeeper::tryRemoveRecursive(const std::string & path) +Coordination::Error ZooKeeper::tryRemoveRecursive(const std::string & path, uint32_t remove_nodes_limit) { - tryRemoveChildrenRecursive(path); - tryRemove(path); -} + if (!isFeatureEnabled(DB::KeeperFeatureFlag::REMOVE_RECURSIVE)) + { + tryRemoveChildrenRecursive(path); + return tryRemove(path); + } + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::RemoveRecursiveResponse & response) mutable + { + promise->set_value(response); + }; + + impl->removeRecursive(path, remove_nodes_limit, std::move(callback)); + + if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready) + { + impl->finalize(fmt::format("Operation timeout on {} {}", Coordination::OpNum::RemoveRecursive, path)); + return Coordination::Error::ZOPERATIONTIMEOUT; + } + else + { + auto response = future.get(); + return response.error; + } +} namespace { @@ -1619,6 +1648,14 @@ Coordination::RequestPtr makeRemoveRequest(const std::string & path, int version return request; } +Coordination::RequestPtr makeRemoveRecursiveRequest(const std::string & path, uint32_t remove_nodes_limit) +{ + auto request = std::make_shared(); + request->path = path; + request->remove_nodes_limit = remove_nodes_limit; + return request; +} + Coordination::RequestPtr makeSetRequest(const std::string & path, const std::string & data, int version) { auto request = std::make_shared(); diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 7ccdc9d1b7f..29c4fbc9b74 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -479,15 +479,16 @@ public: Int64 getClientID(); - /// Remove the node with the subtree. If someone concurrently adds or removes a node - /// in the subtree, the result is undefined. - void removeRecursive(const std::string & path); + /// Remove the node with the subtree. + /// If Keeper supports RemoveRecursive operation then it will be performed atomically. + /// Otherwise if someone concurrently adds or removes a node in the subtree, the result is undefined. + void removeRecursive(const std::string & path, uint32_t remove_nodes_limit = 100); - /// Remove the node with the subtree. If someone concurrently removes a node in the subtree, - /// this will not cause errors. + /// Same as removeRecursive but in case if Keeper does not supports RemoveRecursive and + /// if someone concurrently removes a node in the subtree, this will not cause errors. /// For instance, you can call this method twice concurrently for the same node and the end /// result would be the same as for the single call. - void tryRemoveRecursive(const std::string & path); + Coordination::Error tryRemoveRecursive(const std::string & path, uint32_t remove_nodes_limit = 100); /// Similar to removeRecursive(...) and tryRemoveRecursive(...), but does not remove path itself. /// Node defined as RemoveException will not be deleted. diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index dff14f74681..3f9225e84dd 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -1,5 +1,5 @@ -#include "Common/ZooKeeper/IKeeper.h" -#include "Common/ZooKeeper/ZooKeeperConstants.h" +#include +#include #include #include #include @@ -232,6 +232,27 @@ void ZooKeeperRemoveRequest::readImpl(ReadBuffer & in) Coordination::read(version, in); } +void ZooKeeperRemoveRecursiveRequest::writeImpl(WriteBuffer & out) const +{ + Coordination::write(path, out); + Coordination::write(remove_nodes_limit, out); +} + +void ZooKeeperRemoveRecursiveRequest::readImpl(ReadBuffer & in) +{ + Coordination::read(path, in); + Coordination::read(remove_nodes_limit, in); +} + +std::string ZooKeeperRemoveRecursiveRequest::toStringImpl(bool /*short_format*/) const +{ + return fmt::format( + "path = {}\n" + "remove_nodes_limit = {}", + path, + remove_nodes_limit); +} + void ZooKeeperExistsRequest::writeImpl(WriteBuffer & out) const { Coordination::write(path, out); @@ -510,6 +531,11 @@ ZooKeeperMultiRequest::ZooKeeperMultiRequest(std::span(*concrete_request_remove)); } + else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) + { + checkOperationType(Write); + requests.push_back(std::make_shared(*concrete_request_remove_recursive)); + } else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) { checkOperationType(Write); @@ -707,6 +733,7 @@ ZooKeeperResponsePtr ZooKeeperHeartbeatRequest::makeResponse() const { return se ZooKeeperResponsePtr ZooKeeperSyncRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperAuthRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperRemoveRequest::makeResponse() const { return setTime(std::make_shared()); } +ZooKeeperResponsePtr ZooKeeperRemoveRecursiveRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperExistsRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return setTime(std::make_shared()); } @@ -1024,6 +1051,7 @@ ZooKeeperRequestFactory::ZooKeeperRequestFactory() registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); } PathMatchResult matchPath(std::string_view path, std::string_view match_to) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index fd6ec3cd375..66c075b277b 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -285,6 +285,31 @@ struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse size_t bytesSize() const override { return RemoveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); } }; +struct ZooKeeperRemoveRecursiveRequest final : RemoveRecursiveRequest, ZooKeeperRequest +{ + ZooKeeperRemoveRecursiveRequest() = default; + explicit ZooKeeperRemoveRecursiveRequest(const RemoveRecursiveRequest & base) : RemoveRecursiveRequest(base) {} + + OpNum getOpNum() const override { return OpNum::RemoveRecursive; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + std::string toStringImpl(bool short_format) const override; + + ZooKeeperResponsePtr makeResponse() const override; + bool isReadRequest() const override { return false; } + + size_t bytesSize() const override { return RemoveRecursiveRequest::bytesSize() + sizeof(xid); } +}; + +struct ZooKeeperRemoveRecursiveResponse : RemoveRecursiveResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override {} + void writeImpl(WriteBuffer &) const override {} + OpNum getOpNum() const override { return OpNum::RemoveRecursive; } + + size_t bytesSize() const override { return RemoveRecursiveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); } +}; + struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest { ZooKeeperExistsRequest() = default; diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.cpp b/src/Common/ZooKeeper/ZooKeeperConstants.cpp index cf8ba35e992..a2780dfd5e2 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.cpp +++ b/src/Common/ZooKeeper/ZooKeeperConstants.cpp @@ -29,6 +29,7 @@ static const std::unordered_set VALID_OPERATIONS = static_cast(OpNum::GetACL), static_cast(OpNum::FilteredList), static_cast(OpNum::CheckNotExists), + static_cast(OpNum::RemoveRecursive), }; OpNum getOpNum(int32_t raw_op_num) diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 1d9830505f8..9d8e2d4f857 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -40,6 +40,7 @@ enum class OpNum : int32_t FilteredList = 500, CheckNotExists = 501, CreateIfNotExists = 502, + RemoveRecursive = 503, SessionID = 997, /// Special internal request }; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index ba622f30c91..a6dd9738e17 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1347,6 +1347,25 @@ void ZooKeeper::remove( ProfileEvents::increment(ProfileEvents::ZooKeeperRemove); } +void ZooKeeper::removeRecursive( + const String &path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) +{ + if (!isFeatureEnabled(KeeperFeatureFlag::REMOVE_RECURSIVE)) + throw Exception::fromMessage(Error::ZBADARGUMENTS, "RemoveRecursive request type cannot be used because it's not supported by the server"); + + ZooKeeperRemoveRecursiveRequest request; + request.path = path; + request.remove_nodes_limit = remove_nodes_limit; + + RequestInfo request_info; + request_info.request = std::make_shared(std::move(request)); + request_info.callback = [callback](const Response & response) { callback(dynamic_cast(response)); }; + + pushRequest(std::move(request_info)); + ProfileEvents::increment(ProfileEvents::ZooKeeperRemove); +} void ZooKeeper::exists( const String & path, diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 39082cd14c1..47d2ab8f401 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -146,6 +146,11 @@ public: int32_t version, RemoveCallback callback) override; + void removeRecursive( + const String &path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) override; + void exists( const String & path, ExistsCallback callback, diff --git a/src/Common/atomicRename.cpp b/src/Common/atomicRename.cpp index 4acdff5f66c..7d181d72154 100644 --- a/src/Common/atomicRename.cpp +++ b/src/Common/atomicRename.cpp @@ -57,11 +57,13 @@ namespace ErrorCodes namespace DB { -static bool supportsAtomicRenameImpl() +static std::optional supportsAtomicRenameImpl() { VersionNumber renameat2_minimal_version(3, 15, 0); VersionNumber linux_version(Poco::Environment::osVersion()); - return linux_version >= renameat2_minimal_version; + if (linux_version >= renameat2_minimal_version) + return std::nullopt; + return fmt::format("Linux kernel 3.15+ is required, got {}", linux_version.toString()); } static bool renameat2(const std::string & old_path, const std::string & new_path, int flags) @@ -97,10 +99,14 @@ static bool renameat2(const std::string & old_path, const std::string & new_path ErrnoException::throwFromPath(ErrorCodes::SYSTEM_ERROR, new_path, "Cannot rename {} to {}", old_path, new_path); } -bool supportsAtomicRename() +bool supportsAtomicRename(std::string * out_message) { - static bool supports = supportsAtomicRenameImpl(); - return supports; + static auto error = supportsAtomicRenameImpl(); + if (!error.has_value()) + return true; + if (out_message) + *out_message = error.value(); + return false; } } @@ -152,16 +158,22 @@ static bool renameat2(const std::string & old_path, const std::string & new_path } -static bool supportsAtomicRenameImpl() +static std::optional supportsAtomicRenameImpl() { auto fun = dlsym(RTLD_DEFAULT, "renamex_np"); - return fun != nullptr; + if (fun != nullptr) + return std::nullopt; + return "macOS 10.12 or later is required"; } -bool supportsAtomicRename() +bool supportsAtomicRename(std::string * out_message) { - static bool supports = supportsAtomicRenameImpl(); - return supports; + static auto error = supportsAtomicRenameImpl(); + if (!error.has_value()) + return true; + if (out_message) + *out_message = error.value(); + return false; } } @@ -179,8 +191,10 @@ static bool renameat2(const std::string &, const std::string &, int) return false; } -bool supportsAtomicRename() +bool supportsAtomicRename(std::string * out_message) { + if (out_message) + *out_message = "only Linux and macOS are supported"; return false; } diff --git a/src/Common/atomicRename.h b/src/Common/atomicRename.h index 6da8a8f623b..96d0d6d1e5a 100644 --- a/src/Common/atomicRename.h +++ b/src/Common/atomicRename.h @@ -6,7 +6,7 @@ namespace DB { /// Returns true, if the following functions supported by the system -bool supportsAtomicRename(); +bool supportsAtomicRename(std::string * out_message = nullptr); /// Atomically rename old_path to new_path. If new_path exists, do not overwrite it and throw exception void renameNoReplace(const std::string & old_path, const std::string & new_path); diff --git a/src/Common/benchmarks/CMakeLists.txt b/src/Common/benchmarks/CMakeLists.txt index 57ed837db8b..690ced1eb88 100644 --- a/src/Common/benchmarks/CMakeLists.txt +++ b/src/Common/benchmarks/CMakeLists.txt @@ -1,4 +1,4 @@ -clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp) +clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp orc_string_dictionary.cpp) target_link_libraries (integer_hash_tables_and_hashes PRIVATE ch_contrib::gbenchmark_all dbms @@ -7,3 +7,8 @@ target_link_libraries (integer_hash_tables_and_hashes PRIVATE ch_contrib::wyhash ch_contrib::farmhash ch_contrib::xxHash) + +clickhouse_add_executable(orc_string_dictionary orc_string_dictionary.cpp) +target_link_libraries (orc_string_dictionary PRIVATE + ch_contrib::gbenchmark_all + dbms) diff --git a/src/Common/benchmarks/orc_string_dictionary.cpp b/src/Common/benchmarks/orc_string_dictionary.cpp new file mode 100644 index 00000000000..a542ed2182b --- /dev/null +++ b/src/Common/benchmarks/orc_string_dictionary.cpp @@ -0,0 +1,311 @@ +#include +#include +#include + +class OldSortedStringDictionary +{ +public: + struct DictEntry + { + DictEntry(const char * str, size_t len) : data(str), length(len) { } + const char * data; + size_t length; + }; + + OldSortedStringDictionary() : totalLength(0) { } + + // insert a new string into dictionary, return its insertion order + size_t insert(const char * str, size_t len); + + // reorder input index buffer from insertion order to dictionary order + void reorder(std::vector & idxBuffer) const; + + // get dict entries in insertion order + void getEntriesInInsertionOrder(std::vector &) const; + + size_t size() const; + + // return total length of strings in the dictionary + uint64_t length() const; + + void clear(); + + // store indexes of insertion order in the dictionary for not-null rows + std::vector idxInDictBuffer; + +private: + struct LessThan + { + bool operator()(const DictEntry & left, const DictEntry & right) const + { + int ret = memcmp(left.data, right.data, std::min(left.length, right.length)); + if (ret != 0) + { + return ret < 0; + } + return left.length < right.length; + } + }; + + std::map dict; + std::vector> data; + uint64_t totalLength; +}; + +// insert a new string into dictionary, return its insertion order +size_t OldSortedStringDictionary::insert(const char * str, size_t len) +{ + auto ret = dict.insert({DictEntry(str, len), dict.size()}); + if (ret.second) + { + // make a copy to internal storage + data.push_back(std::vector(len)); + memcpy(data.back().data(), str, len); + // update dictionary entry to link pointer to internal storage + DictEntry * entry = const_cast(&(ret.first->first)); + entry->data = data.back().data(); + totalLength += len; + } + return ret.first->second; +} + +/** + * Reorder input index buffer from insertion order to dictionary order + * + * We require this function because string values are buffered by indexes + * in their insertion order. Until the entire dictionary is complete can + * we get their sorted indexes in the dictionary in that ORC specification + * demands dictionary should be ordered. Therefore this function transforms + * the indexes from insertion order to dictionary value order for final + * output. + */ +void OldSortedStringDictionary::reorder(std::vector & idxBuffer) const +{ + // iterate the dictionary to get mapping from insertion order to value order + std::vector mapping(dict.size()); + size_t dictIdx = 0; + for (auto it = dict.cbegin(); it != dict.cend(); ++it) + { + mapping[it->second] = dictIdx++; + } + + // do the transformation + for (size_t i = 0; i != idxBuffer.size(); ++i) + { + idxBuffer[i] = static_cast(mapping[static_cast(idxBuffer[i])]); + } +} + +// get dict entries in insertion order +void OldSortedStringDictionary::getEntriesInInsertionOrder(std::vector & entries) const +{ + entries.resize(dict.size()); + for (auto it = dict.cbegin(); it != dict.cend(); ++it) + { + entries[it->second] = &(it->first); + } +} + +// return count of entries +size_t OldSortedStringDictionary::size() const +{ + return dict.size(); +} + +// return total length of strings in the dictionary +uint64_t OldSortedStringDictionary::length() const +{ + return totalLength; +} + +void OldSortedStringDictionary::clear() +{ + totalLength = 0; + data.clear(); + dict.clear(); +} + + +/** + * Implementation of increasing sorted string dictionary + */ +class NewSortedStringDictionary +{ +public: + struct DictEntry + { + DictEntry(const char * str, size_t len) : data(str), length(len) { } + const char * data; + size_t length; + }; + + struct DictEntryWithIndex + { + DictEntryWithIndex(const char * str, size_t len, size_t index_) : entry(str, len), index(index_) { } + DictEntry entry; + size_t index; + }; + + NewSortedStringDictionary() : totalLength_(0) { } + + // insert a new string into dictionary, return its insertion order + size_t insert(const char * str, size_t len); + + // reorder input index buffer from insertion order to dictionary order + void reorder(std::vector & idxBuffer) const; + + // get dict entries in insertion order + void getEntriesInInsertionOrder(std::vector &) const; + + // return count of entries + size_t size() const; + + // return total length of strings in the dictionary + uint64_t length() const; + + void clear(); + + // store indexes of insertion order in the dictionary for not-null rows + std::vector idxInDictBuffer; + +private: + struct LessThan + { + bool operator()(const DictEntryWithIndex & l, const DictEntryWithIndex & r) + { + const auto & left = l.entry; + const auto & right = r.entry; + int ret = memcmp(left.data, right.data, std::min(left.length, right.length)); + if (ret != 0) + { + return ret < 0; + } + return left.length < right.length; + } + }; + + mutable std::vector flatDict_; + std::unordered_map keyToIndex; + uint64_t totalLength_; +}; + +// insert a new string into dictionary, return its insertion order +size_t NewSortedStringDictionary::insert(const char * str, size_t len) +{ + size_t index = flatDict_.size(); + auto ret = keyToIndex.emplace(std::string(str, len), index); + if (ret.second) + { + flatDict_.emplace_back(ret.first->first.data(), ret.first->first.size(), index); + totalLength_ += len; + } + return ret.first->second; +} + +/** + * Reorder input index buffer from insertion order to dictionary order + * + * We require this function because string values are buffered by indexes + * in their insertion order. Until the entire dictionary is complete can + * we get their sorted indexes in the dictionary in that ORC specification + * demands dictionary should be ordered. Therefore this function transforms + * the indexes from insertion order to dictionary value order for final + * output. + */ +void NewSortedStringDictionary::reorder(std::vector & idxBuffer) const +{ + // iterate the dictionary to get mapping from insertion order to value order + std::vector mapping(flatDict_.size()); + for (size_t i = 0; i < flatDict_.size(); ++i) + { + mapping[flatDict_[i].index] = i; + } + + // do the transformation + for (size_t i = 0; i != idxBuffer.size(); ++i) + { + idxBuffer[i] = static_cast(mapping[static_cast(idxBuffer[i])]); + } +} + +// get dict entries in insertion order +void NewSortedStringDictionary::getEntriesInInsertionOrder(std::vector & entries) const +{ + std::sort( + flatDict_.begin(), + flatDict_.end(), + [](const DictEntryWithIndex & left, const DictEntryWithIndex & right) { return left.index < right.index; }); + + entries.resize(flatDict_.size()); + for (size_t i = 0; i < flatDict_.size(); ++i) + { + entries[i] = &(flatDict_[i].entry); + } +} + +// return count of entries +size_t NewSortedStringDictionary::size() const +{ + return flatDict_.size(); +} + +// return total length of strings in the dictionary +uint64_t NewSortedStringDictionary::length() const +{ + return totalLength_; +} + +void NewSortedStringDictionary::clear() +{ + totalLength_ = 0; + keyToIndex.clear(); + flatDict_.clear(); +} + +template +static std::vector mockStrings() +{ + std::vector res(1000000); + for (auto & s : res) + { + s = "test string dictionary " + std::to_string(rand() % cardinality); + } + return res; +} + +template +static NO_INLINE std::unique_ptr createAndWriteStringDictionary(const std::vector & strs) +{ + auto dict = std::make_unique(); + for (const auto & str : strs) + { + auto index = dict->insert(str.data(), str.size()); + dict->idxInDictBuffer.push_back(index); + } + dict->reorder(dict->idxInDictBuffer); + + return dict; +} + +template +static void BM_writeStringDictionary(benchmark::State & state) +{ + auto strs = mockStrings(); + for (auto _ : state) + { + auto dict = createAndWriteStringDictionary(strs); + benchmark::DoNotOptimize(dict); + } +} + +BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 10); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 10); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 100); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 100); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 1000); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 1000); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 10000); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 10000); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 100000); +BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 100000); + diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index e416fadc829..907e87a6d30 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -39,7 +39,7 @@ using Checksum = CityHash_v1_0_2::uint128; /// Validate checksum of data, and if it mismatches, find out possible reason and throw exception. -static void validateChecksum(char * data, size_t size, const Checksum expected_checksum) +static void validateChecksum(char * data, size_t size, const Checksum expected_checksum, bool external_data) { auto calculated_checksum = CityHash_v1_0_2::CityHash128(data, size); if (expected_checksum == calculated_checksum) @@ -64,6 +64,8 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c "this can be caused by disk bit rot. This exception protects ClickHouse " "from data corruption due to hardware failures."; + int error_code = external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CHECKSUM_DOESNT_MATCH; + auto flip_bit = [](char * buf, size_t pos) { buf[pos / 8] ^= 1 << pos % 8; @@ -87,7 +89,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c { message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " << message_hardware_failure; - throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), error_code); } flip_bit(tmp_data, bit_pos); /// Restore @@ -102,10 +104,10 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c { message << ". The mismatch is caused by single bit flip in checksum. " << message_hardware_failure; - throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), error_code); } - throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), error_code); } static void readHeaderAndGetCodecAndSize( @@ -151,7 +153,7 @@ static void readHeaderAndGetCodecAndSize( "Most likely corrupted data.", size_compressed_without_checksum); if (size_compressed_without_checksum < header_size) - throw Exception(ErrorCodes::CORRUPTED_DATA, "Can't decompress data: " + throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, "Can't decompress data: " "the compressed data size ({}, this should include header size) is less than the header size ({})", size_compressed_without_checksum, static_cast(header_size)); } @@ -202,7 +204,7 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed, readBinaryLittleEndian(checksum.low64, checksum_in); readBinaryLittleEndian(checksum.high64, checksum_in); - validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum); + validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum, external_data); } ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum)); @@ -247,7 +249,7 @@ size_t CompressedReadBufferBase::readCompressedDataBlockForAsynchronous(size_t & readBinaryLittleEndian(checksum.low64, checksum_in); readBinaryLittleEndian(checksum.high64, checksum_in); - validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum); + validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum, external_data); } ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum)); @@ -307,7 +309,7 @@ void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_d UInt8 header_size = ICompressionCodec::getHeaderSize(); if (size_compressed_without_checksum < header_size) - throw Exception(ErrorCodes::CORRUPTED_DATA, + throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, "Can't decompress data: the compressed data size ({}, this should include header size) is less than the header size ({})", size_compressed_without_checksum, static_cast(header_size)); diff --git a/src/Coordination/KeeperConstants.h b/src/Coordination/KeeperConstants.h index 08a7c85585a..d984d077872 100644 --- a/src/Coordination/KeeperConstants.h +++ b/src/Coordination/KeeperConstants.h @@ -11,6 +11,7 @@ enum class KeeperApiVersion : uint8_t WITH_FILTERED_LIST, WITH_MULTI_READ, WITH_CHECK_NOT_EXISTS, + WITH_REMOVE_RECURSIVE, }; const String keeper_system_path = "/keeper"; diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 893bb8e6082..142bd3b7c71 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -91,6 +91,12 @@ bool checkIfRequestIncreaseMem(const Coordination::ZooKeeperRequestPtr & request memory_delta -= remove_req.bytesSize(); break; } + case Coordination::OpNum::RemoveRecursive: + { + Coordination::ZooKeeperRemoveRecursiveRequest & remove_req = dynamic_cast(*sub_zk_request); + memory_delta -= remove_req.bytesSize(); + break; + } default: break; } diff --git a/src/Coordination/KeeperFeatureFlags.h b/src/Coordination/KeeperFeatureFlags.h index 4e26ca60736..e70bd50cc88 100644 --- a/src/Coordination/KeeperFeatureFlags.h +++ b/src/Coordination/KeeperFeatureFlags.h @@ -12,6 +12,7 @@ enum class KeeperFeatureFlag : size_t MULTI_READ, CHECK_NOT_EXISTS, CREATE_IF_NOT_EXISTS, + REMOVE_RECURSIVE, }; class KeeperFeatureFlags diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 918f24efb2c..2eada508e22 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -28,6 +28,16 @@ #include #include +#if USE_SSL +# include +# include +# include +# include +# include +# include +# include +#endif + #include #include #include @@ -48,6 +58,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int LOGICAL_ERROR; extern const int INVALID_CONFIG_PARAMETER; + extern const int BAD_ARGUMENTS; } using namespace std::chrono_literals; @@ -56,6 +67,16 @@ namespace { #if USE_SSL + +int callSetCertificate(SSL * ssl, void * arg) +{ + if (!arg) + return -1; + + const CertificateReloader::Data * data = reinterpret_cast(arg); + return setCertificateCallback(ssl, data, getLogger("SSLContext")); +} + void setSSLParams(nuraft::asio_service::options & asio_opts) { const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); @@ -69,18 +90,55 @@ void setSSLParams(nuraft::asio_service::options & asio_opts) if (!config.has(private_key_file_property)) throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set."); - asio_opts.enable_ssl_ = true; - asio_opts.server_cert_file_ = config.getString(certificate_file_property); - asio_opts.server_key_file_ = config.getString(private_key_file_property); + Poco::Net::Context::Params params; + params.certificateFile = config.getString(certificate_file_property); + if (params.certificateFile.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server certificate file in config '{}' is empty", certificate_file_property); + + params.privateKeyFile = config.getString(private_key_file_property); + if (params.privateKeyFile.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server key file in config '{}' is empty", private_key_file_property); + + auto pass_phrase = config.getString("openSSL.server.privateKeyPassphraseHandler.options.password", ""); + auto certificate_data = std::make_shared(params.certificateFile, params.privateKeyFile, pass_phrase); if (config.has(root_ca_file_property)) - asio_opts.root_cert_file_ = config.getString(root_ca_file_property); + params.caLocation = config.getString(root_ca_file_property); - if (config.getBool("openSSL.server.loadDefaultCAFile", false)) - asio_opts.load_default_ca_file_ = true; + params.loadDefaultCAs = config.getBool("openSSL.server.loadDefaultCAFile", false); + params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString("openSSL.server.verificationMode", "none")); - if (config.getString("openSSL.server.verificationMode", "none") == "none") - asio_opts.skip_verification_ = true; + std::string disabled_protocols_list = config.getString("openSSL.server.disableProtocols", ""); + Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY); + int disabled_protocols = 0; + for (const auto & token : dp_tok) + { + if (token == "sslv2") + disabled_protocols |= Poco::Net::Context::PROTO_SSLV2; + else if (token == "sslv3") + disabled_protocols |= Poco::Net::Context::PROTO_SSLV3; + else if (token == "tlsv1") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1; + else if (token == "tlsv1_1") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_1; + else if (token == "tlsv1_2") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2; + } + + asio_opts.ssl_context_provider_server_ = [params, certificate_data, disabled_protocols] + { + Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_SERVER_USE, params); + context.disableProtocols(disabled_protocols); + SSL_CTX * ssl_ctx = context.takeSslContext(); + SSL_CTX_set_cert_cb(ssl_ctx, callSetCertificate, reinterpret_cast(certificate_data.get())); + return ssl_ctx; + }; + + asio_opts.ssl_context_provider_client_ = [ctx_params = std::move(params)] + { + Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_CLIENT_USE, ctx_params); + return context.takeSslContext(); + }; } #endif diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index acdf209baae..63dc39092cf 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -832,6 +832,15 @@ std::shared_ptr KeeperStorage::UncommittedS return tryGetNodeFromStorage(path); } +template +const typename Container::Node * KeeperStorage::UncommittedState::getActualNodeView(StringRef path, const Node & storage_node) const +{ + if (auto node_it = nodes.find(path.toView()); node_it != nodes.end()) + return node_it->second.node.get(); + + return &storage_node; +} + template Coordination::ACLs KeeperStorage::UncommittedState::getACLs(StringRef path) const { @@ -1124,7 +1133,7 @@ struct KeeperStorageRequestProcessor } virtual KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & /*watches*/, KeeperStorageBase::Watches & /*list_watches*/) const + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & /*watches*/, KeeperStorageBase::Watches & /*list_watches*/) const { return {}; } @@ -1241,7 +1250,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::CREATED); } @@ -1462,16 +1471,41 @@ struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProce } }; +namespace +{ + +template +void addUpdateParentPzxidDelta(Storage & storage, std::vector & deltas, int64_t zxid, StringRef path) +{ + auto parent_path = parentNodePath(path); + if (!storage.uncommitted_state.getNode(parent_path)) + return; + + deltas.emplace_back( + std::string{parent_path}, + zxid, + typename Storage::UpdateNodeDelta + { + [zxid](Storage::Node & parent) + { + parent.pzxid = std::max(parent.pzxid, zxid); + } + } + ); +} + +} + template struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestProcessor { + using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; + bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override { return storage.checkACL(parentNodePath(this->zk_request->getPath()), Coordination::ACL::Delete, session_id, is_local); } - using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; - std::vector preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override { @@ -1488,31 +1522,12 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}}; } - const auto update_parent_pzxid = [&]() - { - auto parent_path = parentNodePath(request.path); - if (!storage.uncommitted_state.getNode(parent_path)) - return; - - new_deltas.emplace_back( - std::string{parent_path}, - zxid, - typename Storage::UpdateNodeDelta - { - [zxid](Storage::Node & parent) - { - parent.pzxid = std::max(parent.pzxid, zxid); - } - } - ); - }; - auto node = storage.uncommitted_state.getNode(request.path); if (!node) { if (request.restored_from_zookeeper_log) - update_parent_pzxid(); + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}}; } else if (request.version != -1 && request.version != node->version) @@ -1521,7 +1536,7 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}}; if (request.restored_from_zookeeper_log) - update_parent_pzxid(); + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); new_deltas.emplace_back( std::string{parentNodePath(request.path)}, @@ -1552,12 +1567,318 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr } KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED); } }; +template +struct KeeperStorageRemoveRecursiveRequestProcessor final : public KeeperStorageRequestProcessor +{ + using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; + + bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override + { + return storage.checkACL(parentNodePath(this->zk_request->getPath()), Coordination::ACL::Delete, session_id, is_local); + } + + std::vector + preprocess(Storage & storage, int64_t zxid, int64_t session_id, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override + { + ProfileEvents::increment(ProfileEvents::KeeperRemoveRequest); + Coordination::ZooKeeperRemoveRecursiveRequest & request = dynamic_cast(*this->zk_request); + + std::vector new_deltas; + + if (Coordination::matchPath(request.path, keeper_system_path) != Coordination::PathMatchResult::NOT_MATCH) + { + auto error_msg = fmt::format("Trying to delete an internal Keeper path ({}) which is not allowed", request.path); + + handleSystemNodeModification(keeper_context, error_msg); + return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}}; + } + + auto node = storage.uncommitted_state.getNode(request.path); + + if (!node) + { + if (request.restored_from_zookeeper_log) + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); + + return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}}; + } + + ToDeleteTreeCollector collector(storage, zxid, session_id, request.remove_nodes_limit); + auto collect_status = collector.collect(request.path, *node); + + if (collect_status == ToDeleteTreeCollector::CollectStatus::NoAuth) + return {typename Storage::Delta{zxid, Coordination::Error::ZNOAUTH}}; + + if (collect_status == ToDeleteTreeCollector::CollectStatus::LimitExceeded) + return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}}; + + if (request.restored_from_zookeeper_log) + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); + + auto delete_deltas = collector.extractDeltas(); + + for (const auto & delta : delete_deltas) + { + const auto * remove_delta = std::get_if(&delta.operation); + if (remove_delta && remove_delta->ephemeral_owner) + storage.unregisterEphemeralPath(remove_delta->ephemeral_owner, delta.path); + } + + new_deltas.insert(new_deltas.end(), std::make_move_iterator(delete_deltas.begin()), std::make_move_iterator(delete_deltas.end())); + + digest = storage.calculateNodesDigest(digest, new_deltas); + + return new_deltas; + } + + Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse(); + Coordination::ZooKeeperRemoveRecursiveResponse & response = dynamic_cast(*response_ptr); + + response.error = storage.commit(zxid); + return response_ptr; + } + + KeeperStorageBase::ResponsesForSessions + processWatches(const Storage & storage, int64_t zxid, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + { + /// need to iterate over zxid deltas and update watches for deleted tree. + const auto & deltas = storage.uncommitted_state.deltas; + + KeeperStorageBase::ResponsesForSessions responses; + for (auto it = deltas.rbegin(); it != deltas.rend() && it->zxid == zxid; ++it) + { + const auto * remove_delta = std::get_if(&it->operation); + if (remove_delta) + { + auto new_responses = processWatchesImpl(it->path, watches, list_watches, Coordination::Event::DELETED); + responses.insert(responses.end(), std::make_move_iterator(new_responses.begin()), std::make_move_iterator(new_responses.end())); + } + } + + return responses; + } + +private: + using SNode = typename Storage::Node; + + class ToDeleteTreeCollector + { + Storage & storage; + int64_t zxid; + int64_t session_id; + uint32_t limit; + + uint32_t max_level = 0; + uint32_t nodes_observed = 1; /// root node + std::unordered_map> by_level_deltas; + + struct Step + { + String path; + std::variant node; + uint32_t level; + }; + + enum class CollectStatus + { + Ok, + NoAuth, + LimitExceeded, + }; + + friend struct KeeperStorageRemoveRecursiveRequestProcessor; + + public: + ToDeleteTreeCollector(Storage & storage_, int64_t zxid_, int64_t session_id_, uint32_t limit_) + : storage(storage_) + , zxid(zxid_) + , session_id(session_id_) + , limit(limit_) + { + } + + CollectStatus collect(StringRef root_path, const SNode & root_node) + { + std::deque steps; + + if (checkLimits(&root_node)) + return CollectStatus::LimitExceeded; + + steps.push_back(Step{root_path.toString(), &root_node, 0}); + + while (!steps.empty()) + { + Step step = std::move(steps.front()); + steps.pop_front(); + + StringRef path = step.path; + uint32_t level = step.level; + const SNode * node_ptr = nullptr; + + if (auto * rdb = std::get_if(&step.node)) + node_ptr = rdb; + else + node_ptr = std::get(step.node); + + chassert(!path.empty()); + chassert(node_ptr != nullptr); + + const auto & node = *node_ptr; + auto actual_node_ptr = storage.uncommitted_state.getActualNodeView(path, node); + chassert(actual_node_ptr != nullptr); /// explicitly check that node is not deleted + + if (actual_node_ptr->numChildren() > 0 && !storage.checkACL(path, Coordination::ACL::Delete, session_id, /*is_local=*/false)) + return CollectStatus::NoAuth; + + if (auto status = visitRocksDBNode(steps, path, level); status != CollectStatus::Ok) + return status; + + if (auto status = visitMemNode(steps, path, level); status != CollectStatus::Ok) + return status; + + if (auto status = visitRootAndUncommitted(steps, path, node, level); status != CollectStatus::Ok) + return status; + } + + return CollectStatus::Ok; + } + + std::vector extractDeltas() + { + std::vector deltas; + + for (ssize_t level = max_level; level >= 0; --level) + { + auto & level_deltas = by_level_deltas[static_cast(level)]; + deltas.insert(deltas.end(), std::make_move_iterator(level_deltas.begin()), std::make_move_iterator(level_deltas.end())); + } + + return std::move(deltas); + } + + private: + CollectStatus visitRocksDBNode(std::deque & steps, StringRef root_path, uint32_t level) + { + if constexpr (Storage::use_rocksdb) + { + std::filesystem::path root_fs_path(root_path.toString()); + auto children = storage.container.getChildren(root_path.toString()); + + for (auto && [child_name, child_node] : children) + { + auto child_path = (root_fs_path / child_name).generic_string(); + const auto actual_child_node_ptr = storage.uncommitted_state.getActualNodeView(child_path, child_node); + + if (actual_child_node_ptr == nullptr) /// node was deleted in previous step of multi transaction + continue; + + if (checkLimits(actual_child_node_ptr)) + return CollectStatus::LimitExceeded; + + steps.push_back(Step{std::move(child_path), std::move(child_node), level + 1}); + } + } + + return CollectStatus::Ok; + } + + CollectStatus visitMemNode(std::deque & steps, StringRef root_path, uint32_t level) + { + if constexpr (!Storage::use_rocksdb) + { + auto node_it = storage.container.find(root_path); + if (node_it == storage.container.end()) + return CollectStatus::Ok; + + std::filesystem::path root_fs_path(root_path.toString()); + const auto & children = node_it->value.getChildren(); + + for (const auto & child_name : children) + { + auto child_path = (root_fs_path / child_name.toView()).generic_string(); + + auto child_it = storage.container.find(child_path); + chassert(child_it != storage.container.end()); + const auto & child_node = child_it->value; + + const auto actual_child_node_ptr = storage.uncommitted_state.getActualNodeView(child_path, child_node); + + if (actual_child_node_ptr == nullptr) /// node was deleted in previous step of multi transaction + continue; + + if (checkLimits(actual_child_node_ptr)) + return CollectStatus::LimitExceeded; + + steps.push_back(Step{std::move(child_path), &child_node, level + 1}); + } + } + + return CollectStatus::Ok; + } + + CollectStatus visitRootAndUncommitted(std::deque & steps, StringRef root_path, const SNode & root_node, uint32_t level) + { + const auto & nodes = storage.uncommitted_state.nodes; + + /// nodes are sorted by paths with level locality + auto it = nodes.upper_bound(root_path.toString() + "/"); + + for (; it != nodes.end() && parentNodePath(it->first) == root_path; ++it) + { + const auto actual_child_node_ptr = it->second.node.get(); + + if (actual_child_node_ptr == nullptr) /// node was deleted in previous step of multi transaction + continue; + + if (checkLimits(actual_child_node_ptr)) + return CollectStatus::LimitExceeded; + + const String & child_path = it->first; + const SNode & child_node = *it->second.node; + + steps.push_back(Step{child_path, &child_node, level + 1}); + } + + addDelta(root_path, root_node, level); + + return CollectStatus::Ok; + } + + void addDelta(StringRef root_path, const SNode & root_node, uint32_t level) + { + max_level = std::max(max_level, level); + + by_level_deltas[level].emplace_back( + parentNodePath(root_path).toString(), + zxid, + typename Storage::UpdateNodeDelta{ + [](SNode & parent) + { + ++parent.cversion; + parent.decreaseNumChildren(); + } + }); + + by_level_deltas[level].emplace_back(root_path.toString(), zxid, typename Storage::RemoveNodeDelta{root_node.version, root_node.ephemeralOwner()}); + } + + bool checkLimits(const SNode * node) + { + chassert(node != nullptr); + nodes_observed += node->numChildren(); + return nodes_observed > limit; + } + }; +}; + template struct KeeperStorageExistsRequestProcessor final : public KeeperStorageRequestProcessor { @@ -1709,7 +2030,7 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce } KeeperStorageBase::ResponsesForSessions - processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::CHANGED); } @@ -2131,6 +2452,10 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro check_operation_type(OperationType::Write); concrete_requests.push_back(std::make_shared>(sub_zk_request)); break; + case Coordination::OpNum::RemoveRecursive: + check_operation_type(OperationType::Write); + concrete_requests.push_back(std::make_shared>(sub_zk_request)); + break; case Coordination::OpNum::Set: check_operation_type(OperationType::Write); concrete_requests.push_back(std::make_shared>(sub_zk_request)); @@ -2220,6 +2545,7 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro response.responses[i]->error = failed_multi->error_codes[i]; } + response.error = failed_multi->global_error; storage.uncommitted_state.commit(zxid); return response_ptr; } @@ -2250,12 +2576,12 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro } KeeperStorageBase::ResponsesForSessions - processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override + processWatches(const Storage & storage, int64_t zxid, typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override { typename Storage::ResponsesForSessions result; for (const auto & generic_request : concrete_requests) { - auto responses = generic_request->processWatches(watches, list_watches); + auto responses = generic_request->processWatches(storage, zxid, watches, list_watches); result.insert(result.end(), responses.begin(), responses.end()); } return result; @@ -2400,6 +2726,7 @@ KeeperStorageRequestProcessorsFactory::KeeperStorageRequestProcessorsFa registerKeeperRequestProcessor>(*this); registerKeeperRequestProcessor>(*this); registerKeeperRequestProcessor>(*this); + registerKeeperRequestProcessor>(*this); } @@ -2575,7 +2902,19 @@ void KeeperStorage::preprocessRequest( if (check_acl && !request_processor->checkAuth(*this, session_id, false)) { - uncommitted_state.deltas.emplace_back(new_last_zxid, Coordination::Error::ZNOAUTH); + /// Multi requests handle failures using FailedMultiDelta + if (zk_request->getOpNum() == Coordination::OpNum::Multi || zk_request->getOpNum() == Coordination::OpNum::MultiRead) + { + const auto & multi_request = dynamic_cast(*zk_request); + std::vector response_errors; + response_errors.resize(multi_request.requests.size(), Coordination::Error::ZOK); + uncommitted_state.deltas.emplace_back( + new_last_zxid, KeeperStorage::FailedMultiDelta{std::move(response_errors), Coordination::Error::ZNOAUTH}); + } + else + { + uncommitted_state.deltas.emplace_back(new_last_zxid, Coordination::Error::ZNOAUTH); + } return; } @@ -2718,7 +3057,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::process /// If this requests processed successfully we need to check watches if (response->error == Coordination::Error::ZOK) { - auto watch_responses = request_processor->processWatches(watches, list_watches); + auto watch_responses = request_processor->processWatches(*this, zxid, watches, list_watches); results.insert(results.end(), watch_responses.begin(), watch_responses.end()); } diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 4a9286d4835..6fbc4c2b168 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -522,6 +522,7 @@ public: struct FailedMultiDelta { std::vector error_codes; + Coordination::Error global_error{Coordination::Error::ZOK}; }; // Denotes end of a subrequest in multi request @@ -566,6 +567,7 @@ public: void rollback(int64_t rollback_zxid); std::shared_ptr getNode(StringRef path) const; + const Node * getActualNodeView(StringRef path, const Node & storage_node) const; Coordination::ACLs getACLs(StringRef path) const; void applyDelta(const Delta & delta); @@ -609,7 +611,18 @@ public: using is_transparent = void; // required to make find() work with different type than key_type }; - mutable std::unordered_map nodes; + struct PathCmp + { + using is_transparent = std::true_type; + + auto operator()(const std::string_view a, + const std::string_view b) const + { + return a.size() < b.size() || (a.size() == b.size() && a < b); + } + }; + + mutable std::map nodes; std::unordered_map, Hash, Equal> deltas_for_path; std::list deltas; diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index d39031773cd..46f36fe0039 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -2280,6 +2280,62 @@ TYPED_TEST(CoordinationTest, TestPreprocessWhenCloseSessionIsPrecommitted) } } +TYPED_TEST(CoordinationTest, TestMultiRequestWithNoAuth) +{ + using namespace Coordination; + using namespace DB; + + ChangelogDirTest snapshots("./snapshots"); + this->setSnapshotDirectory("./snapshots"); + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + ResponsesQueue queue(std::numeric_limits::max()); + SnapshotsQueue snapshots_queue{1}; + int64_t session_without_auth = 1; + int64_t session_with_auth = 2; + size_t term = 0; + + auto state_machine = std::make_shared>(queue, snapshots_queue, this->keeper_context, nullptr); + state_machine->init(); + + auto & storage = state_machine->getStorageUnsafe(); + + auto auth_req = std::make_shared(); + auth_req->scheme = "digest"; + auth_req->data = "test_user:test_password"; + + // Add auth data to the session + auto auth_entry = getLogEntryFromZKRequest(term, session_with_auth, state_machine->getNextZxid(), auth_req); + state_machine->pre_commit(1, auth_entry->get_buf()); + state_machine->commit(1, auth_entry->get_buf()); + + std::string node_with_acl = "/node_with_acl"; + { + auto create_req = std::make_shared(); + create_req->path = node_with_acl; + create_req->data = "notmodified"; + create_req->acls = {{.permissions = ACL::Read, .scheme = "auth", .id = ""}}; + auto create_entry = getLogEntryFromZKRequest(term, session_with_auth, state_machine->getNextZxid(), create_req); + state_machine->pre_commit(3, create_entry->get_buf()); + state_machine->commit(3, create_entry->get_buf()); + ASSERT_TRUE(storage.container.contains(node_with_acl)); + } + Requests ops; + ops.push_back(zkutil::makeSetRequest(node_with_acl, "modified", -1)); + ops.push_back(zkutil::makeCheckRequest("/nonexistentnode", -1)); + auto multi_req = std::make_shared(ops, ACLs{}); + auto multi_entry = getLogEntryFromZKRequest(term, session_without_auth, state_machine->getNextZxid(), multi_req); + state_machine->pre_commit(4, multi_entry->get_buf()); + state_machine->commit(4, multi_entry->get_buf()); + + auto node_it = storage.container.find(node_with_acl); + ASSERT_FALSE(node_it == storage.container.end()); + ASSERT_TRUE(node_it->value.getData() == "notmodified"); +} + TYPED_TEST(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted) { using namespace Coordination; @@ -3113,6 +3169,8 @@ TYPED_TEST(CoordinationTest, TestFeatureFlags) ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::FILTERED_LIST)); ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::MULTI_READ)); ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CHECK_NOT_EXISTS)); + ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CREATE_IF_NOT_EXISTS)); + ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::REMOVE_RECURSIVE)); } TYPED_TEST(CoordinationTest, TestSystemNodeModify) @@ -3374,6 +3432,474 @@ TYPED_TEST(CoordinationTest, TestReapplyingDeltas) ASSERT_TRUE(children1_set == children2_set); } +TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + + int32_t zxid = 0; + + const auto create = [&](const String & path, int create_mode) + { + int new_zxid = ++zxid; + + const auto create_request = std::make_shared(); + create_request->path = path; + create_request->is_ephemeral = create_mode == zkutil::CreateMode::Ephemeral || create_mode == zkutil::CreateMode::EphemeralSequential; + create_request->is_sequential = create_mode == zkutil::CreateMode::PersistentSequential || create_mode == zkutil::CreateMode::EphemeralSequential; + + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + const auto remove = [&](const String & path, int32_t version = -1) + { + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = path; + remove_request->version = version; + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + return storage.processRequest(remove_request, 1, new_zxid); + }; + + const auto remove_recursive = [&](const String & path, uint32_t remove_nodes_limit = 1) + { + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = path; + remove_request->remove_nodes_limit = remove_nodes_limit; + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + return storage.processRequest(remove_request, 1, new_zxid); + }; + + const auto exists = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto exists_request = std::make_shared(); + exists_request->path = path; + + storage.preprocessRequest(exists_request, 1, 0, new_zxid); + auto responses = storage.processRequest(exists_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + return responses[0].response->error == Coordination::Error::ZOK; + }; + + { + SCOPED_TRACE("Single Remove Single Node"); + create("/T1", zkutil::CreateMode::Persistent); + + auto responses = remove("/T1"); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T1")); + } + + { + SCOPED_TRACE("Single Remove Tree"); + create("/T2", zkutil::CreateMode::Persistent); + create("/T2/A", zkutil::CreateMode::Persistent); + + auto responses = remove("/T2"); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); + ASSERT_TRUE(exists("/T2")); + } + + { + SCOPED_TRACE("Recursive Remove Single Node"); + create("/T3", zkutil::CreateMode::Persistent); + + auto responses = remove_recursive("/T3", 100); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T3")); + } + + { + SCOPED_TRACE("Recursive Remove Tree Small Limit"); + create("/T5", zkutil::CreateMode::Persistent); + create("/T5/A", zkutil::CreateMode::Persistent); + create("/T5/B", zkutil::CreateMode::Persistent); + create("/T5/A/C", zkutil::CreateMode::Persistent); + + auto responses = remove_recursive("/T5", 2); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); + ASSERT_TRUE(exists("/T5")); + ASSERT_TRUE(exists("/T5/A")); + ASSERT_TRUE(exists("/T5/B")); + ASSERT_TRUE(exists("/T5/A/C")); + } + + { + SCOPED_TRACE("Recursive Remove Tree Big Limit"); + create("/T6", zkutil::CreateMode::Persistent); + create("/T6/A", zkutil::CreateMode::Persistent); + create("/T6/B", zkutil::CreateMode::Persistent); + create("/T6/A/C", zkutil::CreateMode::Persistent); + + auto responses = remove_recursive("/T6", 4); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T6")); + ASSERT_FALSE(exists("/T6/A")); + ASSERT_FALSE(exists("/T6/B")); + ASSERT_FALSE(exists("/T6/A/C")); + } + + { + SCOPED_TRACE("Recursive Remove Ephemeral"); + create("/T7", zkutil::CreateMode::Ephemeral); + ASSERT_EQ(storage.ephemerals.size(), 1); + + auto responses = remove_recursive("/T7", 100); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_EQ(storage.ephemerals.size(), 0); + ASSERT_FALSE(exists("/T7")); + } + + { + SCOPED_TRACE("Recursive Remove Tree With Ephemeral"); + create("/T8", zkutil::CreateMode::Persistent); + create("/T8/A", zkutil::CreateMode::Persistent); + create("/T8/B", zkutil::CreateMode::Ephemeral); + create("/T8/A/C", zkutil::CreateMode::Ephemeral); + ASSERT_EQ(storage.ephemerals.size(), 1); + + auto responses = remove_recursive("/T8", 4); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_EQ(storage.ephemerals.size(), 0); + ASSERT_FALSE(exists("/T8")); + ASSERT_FALSE(exists("/T8/A")); + ASSERT_FALSE(exists("/T8/B")); + ASSERT_FALSE(exists("/T8/A/C")); + } +} + +TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + int zxid = 0; + + auto prepare_create_tree = []() + { + return Coordination::Requests{ + zkutil::makeCreateRequest("/A", "A", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest("/A/B", "B", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest("/A/C", "C", zkutil::CreateMode::Ephemeral), + zkutil::makeCreateRequest("/A/B/D", "D", zkutil::CreateMode::Ephemeral), + }; + }; + + const auto exists = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto exists_request = std::make_shared(); + exists_request->path = path; + + storage.preprocessRequest(exists_request, 1, 0, new_zxid); + auto responses = storage.processRequest(exists_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + return responses[0].response->error == Coordination::Error::ZOK; + }; + + const auto is_multi_ok = [&](Coordination::ZooKeeperResponsePtr response) + { + const auto & multi_response = dynamic_cast(*response); + + for (const auto & op_response : multi_response.responses) + if (op_response->error != Coordination::Error::ZOK) + return false; + + return true; + }; + + { + SCOPED_TRACE("Remove In Multi Tx"); + int new_zxid = ++zxid; + auto ops = prepare_create_tree(); + + ops.push_back(zkutil::makeRemoveRequest("/A", -1)); + const auto request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(request, 1, 0, new_zxid); + auto responses = storage.processRequest(request, 1, new_zxid); + ops.pop_back(); + + ASSERT_EQ(responses.size(), 1); + ASSERT_FALSE(is_multi_ok(responses[0].response)); + } + + { + SCOPED_TRACE("Recursive Remove In Multi Tx"); + int new_zxid = ++zxid; + auto ops = prepare_create_tree(); + + ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 4)); + const auto request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(request, 1, 0, new_zxid); + auto responses = storage.processRequest(request, 1, new_zxid); + ops.pop_back(); + + ASSERT_EQ(responses.size(), 1); + ASSERT_TRUE(is_multi_ok(responses[0].response)); + ASSERT_FALSE(exists("/A")); + ASSERT_FALSE(exists("/A/C")); + ASSERT_FALSE(exists("/A/B")); + ASSERT_FALSE(exists("/A/B/D")); + } + + { + SCOPED_TRACE("Recursive Remove With Regular In Multi Tx"); + int new_zxid = ++zxid; + auto ops = prepare_create_tree(); + + ops.push_back(zkutil::makeRemoveRequest("/A/C", -1)); + ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 3)); + const auto request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(request, 1, 0, new_zxid); + auto responses = storage.processRequest(request, 1, new_zxid); + ops.pop_back(); + ops.pop_back(); + + ASSERT_EQ(responses.size(), 1); + ASSERT_TRUE(is_multi_ok(responses[0].response)); + ASSERT_FALSE(exists("/A")); + ASSERT_FALSE(exists("/A/C")); + ASSERT_FALSE(exists("/A/B")); + ASSERT_FALSE(exists("/A/B/D")); + } + + { + SCOPED_TRACE("Recursive Remove From Committed and Uncommitted states"); + int create_zxid = ++zxid; + auto ops = prepare_create_tree(); + + /// First create nodes + const auto create_request = std::make_shared(ops, ACLs{}); + storage.preprocessRequest(create_request, 1, 0, create_zxid); + auto create_responses = storage.processRequest(create_request, 1, create_zxid); + ASSERT_EQ(create_responses.size(), 1); + ASSERT_TRUE(is_multi_ok(create_responses[0].response)); + ASSERT_TRUE(exists("/A")); + ASSERT_TRUE(exists("/A/C")); + ASSERT_TRUE(exists("/A/B")); + ASSERT_TRUE(exists("/A/B/D")); + + /// Remove node A/C as a single remove request. + /// Remove all other as remove recursive request. + /// In this case we should list storage to understand the tree topology + /// but ignore already deleted nodes in uncommitted state. + + int remove_zxid = ++zxid; + ops = { + zkutil::makeRemoveRequest("/A/C", -1), + zkutil::makeRemoveRecursiveRequest("/A", 3), + }; + const auto remove_request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(remove_request, 1, 0, remove_zxid); + auto remove_responses = storage.processRequest(remove_request, 1, remove_zxid); + + ASSERT_EQ(remove_responses.size(), 1); + ASSERT_TRUE(is_multi_ok(remove_responses[0].response)); + ASSERT_FALSE(exists("/A")); + ASSERT_FALSE(exists("/A/C")); + ASSERT_FALSE(exists("/A/B")); + ASSERT_FALSE(exists("/A/B/D")); + } +} + +TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + int zxid = 0; + + const auto create = [&](const String & path, int create_mode) + { + int new_zxid = ++zxid; + + const auto create_request = std::make_shared(); + create_request->path = path; + create_request->is_ephemeral = create_mode == zkutil::CreateMode::Ephemeral || create_mode == zkutil::CreateMode::EphemeralSequential; + create_request->is_sequential = create_mode == zkutil::CreateMode::PersistentSequential || create_mode == zkutil::CreateMode::EphemeralSequential; + + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + const auto add_watch = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto exists_request = std::make_shared(); + exists_request->path = path; + exists_request->has_watch = true; + + storage.preprocessRequest(exists_request, 1, 0, new_zxid); + auto responses = storage.processRequest(exists_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK); + }; + + const auto add_list_watch = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto list_request = std::make_shared(); + list_request->path = path; + list_request->has_watch = true; + + storage.preprocessRequest(list_request, 1, 0, new_zxid); + auto responses = storage.processRequest(list_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK); + }; + + create("/A", zkutil::CreateMode::Persistent); + create("/A/B", zkutil::CreateMode::Persistent); + create("/A/C", zkutil::CreateMode::Ephemeral); + create("/A/B/D", zkutil::CreateMode::Ephemeral); + + add_watch("/A"); + add_watch("/A/B"); + add_watch("/A/C"); + add_watch("/A/B/D"); + add_list_watch("/A"); + add_list_watch("/A/B"); + ASSERT_EQ(storage.watches.size(), 4); + ASSERT_EQ(storage.list_watches.size(), 2); + + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = "/A"; + remove_request->remove_nodes_limit = 4; + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + auto responses = storage.processRequest(remove_request, 1, new_zxid); + + ASSERT_EQ(responses.size(), 7); + + for (size_t i = 0; i < 7; ++i) + { + ASSERT_EQ(responses[i].response->error, Coordination::Error::ZOK); + + if (const auto * watch_response = dynamic_cast(responses[i].response.get())) + ASSERT_EQ(watch_response->type, Coordination::Event::DELETED); + } + + ASSERT_EQ(storage.watches.size(), 0); + ASSERT_EQ(storage.list_watches.size(), 0); +} + +TYPED_TEST(CoordinationTest, TestRemoveRecursiveAcls) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + int zxid = 0; + + { + int new_zxid = ++zxid; + String user_auth_data = "test_user:test_password"; + + const auto auth_request = std::make_shared(); + auth_request->scheme = "digest"; + auth_request->data = user_auth_data; + + storage.preprocessRequest(auth_request, 1, 0, new_zxid); + auto responses = storage.processRequest(auth_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to add auth to session"; + } + + const auto create = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto create_request = std::make_shared(); + create_request->path = path; + create_request->acls = {{.permissions = ACL::Create, .scheme = "auth", .id = ""}}; + + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + /// Add nodes with only Create ACL + create("/A"); + create("/A/B"); + create("/A/C"); + create("/A/B/D"); + + { + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = "/A"; + remove_request->remove_nodes_limit = 4; + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + auto responses = storage.processRequest(remove_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZNOAUTH); + } +} + /// INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, /// CoordinationTest, /// ::testing::ValuesIn(std::initializer_list{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}})); diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 807e4a7187a..5dc9082d49d 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -890,16 +890,19 @@ public: Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - AuthenticationType user_auth_type; try { - user_auth_type = session.getAuthenticationTypeOrLogInFailure(user_name); - if (type_to_method.find(user_auth_type) != type_to_method.end()) + const auto user_authentication_types = session.getAuthenticationTypesOrLogInFailure(user_name); + + for (auto user_authentication_type : user_authentication_types) { - type_to_method[user_auth_type]->authenticate(user_name, session, mt, address); - mt.send(Messaging::AuthenticationOk(), true); - LOG_DEBUG(log, "Authentication for user {} was successful.", user_name); - return; + if (type_to_method.find(user_authentication_type) != type_to_method.end()) + { + type_to_method[user_authentication_type]->authenticate(user_name, session, mt, address); + mt.send(Messaging::AuthenticationOk(), true); + LOG_DEBUG(log, "Authentication for user {} was successful.", user_name); + return; + } } } catch (const Exception&) @@ -913,7 +916,7 @@ public: mt.send(Messaging::ErrorOrNoticeResponse(Messaging::ErrorOrNoticeResponse::ERROR, "0A000", "Authentication method is not supported"), true); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Authentication method is not supported: {}", user_auth_type); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "None of the authentication methods registered for the user are supported"); } }; } diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 790987272fa..49c6fc1dde6 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -33,7 +33,8 @@ static constexpr auto DBMS_MIN_REVISION_WITH_AGGREGATE_FUNCTIONS_VERSIONING = 54 static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = 1; -static constexpr auto DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3; +static constexpr auto DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3; +static constexpr auto DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 4; static constexpr auto DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453; static constexpr auto DBMS_MERGE_TREE_PART_INFO_VERSION = 1; @@ -86,6 +87,8 @@ static constexpr auto DBMS_MIN_REVISION_WITH_ROWS_BEFORE_AGGREGATION = 54469; /// Packets size header static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470; +static constexpr auto DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL = 54471; + /// Version of ClickHouse TCP protocol. /// /// Should be incremented manually on protocol changes. @@ -93,6 +96,6 @@ static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470; /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54470; +static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54471; } diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 18ee096569a..13f8373f1ce 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -50,7 +50,7 @@ namespace DB M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating heavy asynchronous metrics.", 0) \ M(String, default_database, "default", "Default database name.", 0) \ M(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \ - M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting., ", 0) \ + M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting.", 0) \ M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \ M(UInt64, aggregate_function_group_array_max_element_size, 0xFFFFFF, "Max array element size in bytes for groupArray function. This limit is checked at serialization and help to avoid large state size.", 0) \ M(GroupArrayActionWhenLimitReached, aggregate_function_group_array_action_when_limit_is_reached, GroupArrayActionWhenLimitReached::THROW, "Action to execute when max array element size is exceeded in groupArray: `throw` exception, or `discard` extra values", 0) \ @@ -65,6 +65,7 @@ namespace DB M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \ M(Bool, async_insert_queue_flush_on_shutdown, true, "If true queue of asynchronous inserts is flushed on graceful shutdown", 0) \ M(Bool, ignore_empty_sql_security_in_create_view_query, true, "If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. This setting is only necessary for the migration period and will become obsolete in 24.4", 0) \ + M(UInt64, max_build_vector_similarity_index_thread_pool_size, 16, "The maximum number of threads to use to build vector similarity indexes. 0 means all cores.", 0) \ \ /* Database Catalog */ \ M(UInt64, database_atomic_delay_before_drop_table_sec, 8 * 60, "The delay during which a dropped table can be restored using the UNDROP statement. If DROP TABLE ran with a SYNC modifier, the setting is ignored.", 0) \ @@ -118,6 +119,7 @@ namespace DB M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \ M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \ + M(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users. Zero means unlimited", 0) \ M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \ M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 62371141b3a..e367cce5969 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -946,7 +946,7 @@ class IColumn; M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \ M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \ M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \ - M(UInt64, parallel_replicas_mark_segment_size, 128, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing", 0) \ + M(UInt64, parallel_replicas_mark_segment_size, 0, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384]", 0) \ M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as ' :: ' if archive has correct extension", 0) \ M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \ \ @@ -972,7 +972,6 @@ class IColumn; \ M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \ M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \ - \ /** Experimental feature for moving data between shards. */ \ M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \ @@ -1145,6 +1144,7 @@ class IColumn; M(Bool, input_format_try_infer_variants, false, "Try to infer the Variant type in text formats when there is more than one possible type for column/array elements", 0) \ M(Bool, type_json_skip_duplicated_paths, false, "When enabled, during parsing JSON object into JSON type duplicated paths will be ignored and only the first one will be inserted instead of an exception", 0) \ M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \ + M(Bool, input_format_json_empty_as_default, false, "Treat empty fields in JSON input as default values.", 0) \ M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \ @@ -1272,6 +1272,7 @@ class IColumn; M(Bool, output_format_orc_string_as_string, true, "Use ORC String type instead of Binary for String columns", 0) \ M(ORCCompression, output_format_orc_compression_method, "zstd", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \ M(UInt64, output_format_orc_row_index_stride, 10'000, "Target row index stride in ORC output format", 0) \ + M(Double, output_format_orc_dictionary_key_size_threshold, 0.0, "For a string column in ORC output format, if the number of distinct values is greater than this fraction of the total number of non-null rows, turn off dictionary encoding. Otherwise dictionary encoding is enabled", 0) \ \ M(CapnProtoEnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::CapnProtoEnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 471c0fd9d31..c88923011ce 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -71,6 +71,8 @@ static std::initializer_list #include +#include namespace DB { @@ -615,28 +616,49 @@ void SerializationArray::serializeTextJSONPretty(const IColumn & column, size_t } -void SerializationArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +template +ReturnType SerializationArray::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - deserializeTextImpl(column, istr, - [&](IColumn & nested_column) + auto deserialize_nested = [&settings, this](IColumn & nested_column, ReadBuffer & buf) -> ReturnType + { + if constexpr (std::is_same_v) { if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column)) - SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(nested_column, istr, settings, nested); + SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested); else - nested->deserializeTextJSON(nested_column, istr, settings); - }, false); + nested->deserializeTextJSON(nested_column, buf, settings); + } + else + { + if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column)) + return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested); + return nested->tryDeserializeTextJSON(nested_column, buf, settings); + } + }; + + if (settings.json.empty_as_default) + return deserializeTextImpl(column, istr, + [&deserialize_nested, &istr](IColumn & nested_column) -> ReturnType + { + return JSONUtils::deserializeEmpyStringAsDefaultOrNested(nested_column, istr, deserialize_nested); + }, false); + else + return deserializeTextImpl(column, istr, + [&deserialize_nested, &istr](IColumn & nested_column) -> ReturnType + { + return deserialize_nested(nested_column, istr); + }, false); +} + + +void SerializationArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + deserializeTextJSONImpl(column, istr, settings); } bool SerializationArray::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - auto read_nested = [&](IColumn & nested_column) - { - if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column)) - return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(nested_column, istr, settings, nested); - return nested->tryDeserializeTextJSON(nested_column, istr, settings); - }; - - return deserializeTextImpl(column, istr, std::move(read_nested), false); + return deserializeTextJSONImpl(column, istr, settings); } diff --git a/src/DataTypes/Serializations/SerializationArray.h b/src/DataTypes/Serializations/SerializationArray.h index c3353f0c251..7e34abfac90 100644 --- a/src/DataTypes/Serializations/SerializationArray.h +++ b/src/DataTypes/Serializations/SerializationArray.h @@ -82,6 +82,10 @@ public: SerializationPtr create(const SerializationPtr & prev) const override; ColumnPtr create(const ColumnPtr & prev) const override; }; + +private: + template + ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const; }; } diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index c722b3ac7a1..ae864cbf7b4 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -316,28 +317,52 @@ void SerializationMap::serializeTextJSONPretty(const IColumn & column, size_t ro } -void SerializationMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +template +ReturnType SerializationMap::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - deserializeTextImpl(column, istr, - [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) + auto deserialize_nested = [&settings](IColumn & subcolumn, ReadBuffer & buf, const SerializationPtr & subcolumn_serialization) -> ReturnType + { + if constexpr (std::is_same_v) { if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn)) SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization); else subcolumn_serialization->deserializeTextJSON(subcolumn, buf, settings); - }); + } + else + { + if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn)) + return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization); + return subcolumn_serialization->tryDeserializeTextJSON(subcolumn, buf, settings); + } + }; + + if (settings.json.empty_as_default) + return deserializeTextImpl(column, istr, + [&deserialize_nested](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) -> ReturnType + { + return JSONUtils::deserializeEmpyStringAsDefaultOrNested(subcolumn, buf, + [&deserialize_nested, &subcolumn_serialization](IColumn & subcolumn_, ReadBuffer & buf_) -> ReturnType + { + return deserialize_nested(subcolumn_, buf_, subcolumn_serialization); + }); + }); + else + return deserializeTextImpl(column, istr, + [&deserialize_nested](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) -> ReturnType + { + return deserialize_nested(subcolumn, buf, subcolumn_serialization); + }); +} + +void SerializationMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + deserializeTextJSONImpl(column, istr, settings); } bool SerializationMap::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - auto reader = [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) - { - if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(subcolumn)) - return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(subcolumn, buf, settings, subcolumn_serialization); - return subcolumn_serialization->tryDeserializeTextJSON(subcolumn, buf, settings); - }; - - return deserializeTextImpl(column, istr, reader); + return deserializeTextJSONImpl(column, istr, settings); } void SerializationMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/DataTypes/Serializations/SerializationMap.h b/src/DataTypes/Serializations/SerializationMap.h index cfcde445c1f..007d153ec7e 100644 --- a/src/DataTypes/Serializations/SerializationMap.h +++ b/src/DataTypes/Serializations/SerializationMap.h @@ -74,6 +74,9 @@ private: template ReturnType deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && reader) const; + + template + ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const; }; } diff --git a/src/DataTypes/Serializations/SerializationTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp index 594a23ab507..e1fcb1a8d48 100644 --- a/src/DataTypes/Serializations/SerializationTuple.cpp +++ b/src/DataTypes/Serializations/SerializationTuple.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -313,27 +314,9 @@ void SerializationTuple::serializeTextJSONPretty(const IColumn & column, size_t } template -ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +ReturnType SerializationTuple::deserializeTupleJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, auto && deserialize_element) const { - static constexpr bool throw_exception = std::is_same_v; - - auto deserialize_element = [&](IColumn & element_column, size_t element_pos) - { - if constexpr (throw_exception) - { - if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column)) - SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(element_column, istr, settings, elems[element_pos]); - else - elems[element_pos]->deserializeTextJSON(element_column, istr, settings); - return true; - } - else - { - if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column)) - return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(element_column, istr, settings, elems[element_pos]); - return elems[element_pos]->tryDeserializeTextJSON(element_column, istr, settings); - } - }; + static constexpr auto throw_exception = std::is_same_v; if (settings.json.read_named_tuples_as_objects && have_explicit_names) @@ -506,12 +489,51 @@ ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuf } } -void SerializationTuple::deserializeTextJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings & settings) const +template +ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - deserializeTextJSONImpl(column, istr, settings); + auto deserialize_nested = [&settings](IColumn & nested_column, ReadBuffer & buf, const SerializationPtr & nested_column_serialization) -> ReturnType + { + if constexpr (std::is_same_v) + { + if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column)) + SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested_column_serialization); + else + nested_column_serialization->deserializeTextJSON(nested_column, buf, settings); + } + else + { + if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(nested_column)) + return SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(nested_column, buf, settings, nested_column_serialization); + else + return nested_column_serialization->tryDeserializeTextJSON(nested_column, buf, settings); + } + }; + + if (settings.json.empty_as_default) + return deserializeTupleJSONImpl(column, istr, settings, + [&deserialize_nested, &istr, this](IColumn & nested_column, size_t element_pos) -> ReturnType + { + return JSONUtils::deserializeEmpyStringAsDefaultOrNested(nested_column, istr, + [&deserialize_nested, element_pos, this](IColumn & nested_column_, ReadBuffer & buf) -> ReturnType + { + return deserialize_nested(nested_column_, buf, elems[element_pos]); + }); + }); + else + return deserializeTupleJSONImpl(column, istr, settings, + [&deserialize_nested, &istr, this](IColumn & nested_column, size_t element_pos) -> ReturnType + { + return deserialize_nested(nested_column, istr, elems[element_pos]); + }); } -bool SerializationTuple::tryDeserializeTextJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings & settings) const +void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + deserializeTextJSONImpl(column, istr, settings); +} + +bool SerializationTuple::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { return deserializeTextJSONImpl(column, istr, settings); } diff --git a/src/DataTypes/Serializations/SerializationTuple.h b/src/DataTypes/Serializations/SerializationTuple.h index 810673d8b21..c51adb6e536 100644 --- a/src/DataTypes/Serializations/SerializationTuple.h +++ b/src/DataTypes/Serializations/SerializationTuple.h @@ -81,7 +81,10 @@ private: template ReturnType deserializeTextImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const; - template + template + ReturnType deserializeTupleJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, auto && deserialize_element) const; + + template ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const; template diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index d86e29ca915..e2e2414b1ca 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -197,8 +197,9 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Moving tables between databases of different engines is not supported"); } - if (exchange && !supportsAtomicRename()) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RENAME EXCHANGE is not supported"); + std::string message; + if (exchange && !supportsAtomicRename(&message)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RENAME EXCHANGE is not supported because exchanging files is not supported by the OS ({})", message); waitDatabaseStarted(); diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 05a5e057c55..d97474bd245 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -59,35 +59,27 @@ void cckMetadataPathForOrdinary(const ASTCreateQuery & create, const String & me } -/// validate validates the database engine that's specified in the create query for -/// engine arguments, settings and table overrides. -void validate(const ASTCreateQuery & create_query) - +void DatabaseFactory::validate(const ASTCreateQuery & create_query) const { auto * storage = create_query.storage; - /// Check engine may have arguments - static const std::unordered_set engines_with_arguments{"MySQL", "MaterializeMySQL", "MaterializedMySQL", - "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite", "Filesystem", "S3", "HDFS"}; - const String & engine_name = storage->engine->name; - bool engine_may_have_arguments = engines_with_arguments.contains(engine_name); + const EngineFeatures & engine_features = database_engines.at(engine_name).features; - if (storage->engine->arguments && !engine_may_have_arguments) + /// Check engine may have arguments + if (storage->engine->arguments && !engine_features.supports_arguments) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have arguments", engine_name); /// Check engine may have settings - bool may_have_settings = endsWith(engine_name, "MySQL") || engine_name == "Replicated" || engine_name == "MaterializedPostgreSQL"; bool has_unexpected_element = storage->engine->parameters || storage->partition_by || storage->primary_key || storage->order_by || storage->sample_by; - if (has_unexpected_element || (!may_have_settings && storage->settings)) + if (has_unexpected_element || (!engine_features.supports_settings && storage->settings)) throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_AST, "Database engine `{}` cannot have parameters, primary_key, order_by, sample_by, settings", engine_name); /// Check engine with table overrides - static const std::unordered_set engines_with_table_overrides{"MaterializeMySQL", "MaterializedMySQL", "MaterializedPostgreSQL"}; - if (create_query.table_overrides && !engines_with_table_overrides.contains(engine_name)) + if (create_query.table_overrides && !engine_features.supports_table_overrides) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have table overrides", engine_name); } @@ -121,9 +113,9 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m return impl; } -void DatabaseFactory::registerDatabase(const std::string & name, CreatorFn creator_fn) +void DatabaseFactory::registerDatabase(const std::string & name, CreatorFn creator_fn, EngineFeatures features) { - if (!database_engines.emplace(name, std::move(creator_fn)).second) + if (!database_engines.emplace(name, Creator{std::move(creator_fn), features}).second) throw Exception(ErrorCodes::LOGICAL_ERROR, "DatabaseFactory: the database engine name '{}' is not unique", name); } @@ -154,7 +146,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String .context = context}; // creator_fn creates and returns a DatabasePtr with the supplied arguments - auto creator_fn = database_engines.at(engine_name); + auto creator_fn = database_engines.at(engine_name).creator_fn; return creator_fn(arguments); } diff --git a/src/Databases/DatabaseFactory.h b/src/Databases/DatabaseFactory.h index 494c9e0076e..ede4394e435 100644 --- a/src/Databases/DatabaseFactory.h +++ b/src/Databases/DatabaseFactory.h @@ -43,13 +43,30 @@ public: ContextPtr & context; }; - DatabasePtr get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context); + struct EngineFeatures + { + bool supports_arguments = false; + bool supports_settings = false; + bool supports_table_overrides = false; + }; using CreatorFn = std::function; - using DatabaseEngines = std::unordered_map; + struct Creator + { + CreatorFn creator_fn; + EngineFeatures features; + }; - void registerDatabase(const std::string & name, CreatorFn creator_fn); + DatabasePtr get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context); + + using DatabaseEngines = std::unordered_map; + + void registerDatabase(const std::string & name, CreatorFn creator_fn, EngineFeatures features = EngineFeatures{ + .supports_arguments = false, + .supports_settings = false, + .supports_table_overrides = false, + }); const DatabaseEngines & getDatabaseEngines() const { return database_engines; } @@ -65,6 +82,10 @@ private: DatabaseEngines database_engines; DatabasePtr getImpl(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context); + + /// validate validates the database engine that's specified in the create query for + /// engine arguments, settings and table overrides. + void validate(const ASTCreateQuery & create_query) const; }; } diff --git a/src/Databases/DatabaseFilesystem.cpp b/src/Databases/DatabaseFilesystem.cpp index 31701e665a1..4b50e79da4a 100644 --- a/src/Databases/DatabaseFilesystem.cpp +++ b/src/Databases/DatabaseFilesystem.cpp @@ -257,6 +257,6 @@ void registerDatabaseFilesystem(DatabaseFactory & factory) return std::make_shared(args.database_name, init_path, args.context); }; - factory.registerDatabase("Filesystem", create_fn); + factory.registerDatabase("Filesystem", create_fn, {.supports_arguments = true}); } } diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index 7fa67a5678e..ceca2666e49 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -253,7 +253,7 @@ void registerDatabaseHDFS(DatabaseFactory & factory) return std::make_shared(args.database_name, source_url, args.context); }; - factory.registerDatabase("HDFS", create_fn); + factory.registerDatabase("HDFS", create_fn, {.supports_arguments = true}); } } // DB diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 2ccdd8510a8..0a4b02c4917 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -398,6 +398,6 @@ void registerDatabaseLazy(DatabaseFactory & factory) cache_expiration_time_seconds, args.context); }; - factory.registerDatabase("Lazy", create_fn); + factory.registerDatabase("Lazy", create_fn, {.supports_arguments = true}); } } diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 3d64c82ba7d..1bc6334db29 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -2001,6 +2001,6 @@ void registerDatabaseReplicated(DatabaseFactory & factory) replica_name, std::move(database_replicated_settings), args.context); }; - factory.registerDatabase("Replicated", create_fn); + factory.registerDatabase("Replicated", create_fn, {.supports_arguments = true, .supports_settings = true}); } } diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 4e7408aa96e..7993a41df53 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; @@ -249,6 +250,8 @@ String DatabaseReplicatedDDLWorker::enqueueQueryImpl(const ZooKeeperPtr & zookee } else if (code != Coordination::Error::ZNODEEXISTS) zkutil::KeeperMultiException::check(code, ops, res); + + sleepForMilliseconds(50); } if (counter_path.empty()) diff --git a/src/Databases/DatabaseS3.cpp b/src/Databases/DatabaseS3.cpp index 2b2d978a846..d80cc6d0953 100644 --- a/src/Databases/DatabaseS3.cpp +++ b/src/Databases/DatabaseS3.cpp @@ -326,7 +326,7 @@ void registerDatabaseS3(DatabaseFactory & factory) return std::make_shared(args.database_name, config, args.context); }; - factory.registerDatabase("S3", create_fn); + factory.registerDatabase("S3", create_fn, {.supports_arguments = true}); } } #endif diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 2f5477a6b9d..2b728039632 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -290,8 +290,14 @@ void registerDatabaseMaterializedMySQL(DatabaseFactory & factory) binlog_client, std::move(materialize_mode_settings)); }; - factory.registerDatabase("MaterializeMySQL", create_fn); - factory.registerDatabase("MaterializedMySQL", create_fn); + + DatabaseFactory::EngineFeatures features{ + .supports_arguments = true, + .supports_settings = true, + .supports_table_overrides = true, + }; + factory.registerDatabase("MaterializeMySQL", create_fn, features); + factory.registerDatabase("MaterializedMySQL", create_fn, features); } } diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 7aa29018f4d..3b72f2aeae5 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -584,7 +584,7 @@ void registerDatabaseMySQL(DatabaseFactory & factory) throw Exception(ErrorCodes::CANNOT_CREATE_DATABASE, "Cannot create MySQL database, because {}", exception_message); } }; - factory.registerDatabase("MySQL", create_fn); + factory.registerDatabase("MySQL", create_fn, {.supports_arguments = true, .supports_settings = true}); } } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index 6b0548b85c7..ed62398e594 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -546,7 +546,11 @@ void registerDatabaseMaterializedPostgreSQL(DatabaseFactory & factory) args.database_name, configuration.database, connection_info, std::move(postgresql_replica_settings)); }; - factory.registerDatabase("MaterializedPostgreSQL", create_fn); + factory.registerDatabase("MaterializedPostgreSQL", create_fn, { + .supports_arguments = true, + .supports_settings = true, + .supports_table_overrides = true, + }); } } diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 032fc33ea16..0eafd1c3b5b 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -558,7 +558,7 @@ void registerDatabasePostgreSQL(DatabaseFactory & factory) pool, use_table_cache); }; - factory.registerDatabase("PostgreSQL", create_fn); + factory.registerDatabase("PostgreSQL", create_fn, {.supports_arguments = true}); } } diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index b9fd9c325f8..45fd52f27ab 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -292,7 +293,7 @@ PostgreSQLTableStructure::ColumnsInfoPtr readNamesAndTypesList( template PostgreSQLTableStructure fetchPostgreSQLTableStructure( - T & tx, const String & postgres_table, const String & postgres_schema, bool use_nulls, bool with_primary_key, bool with_replica_identity_index) + T & tx, const String & postgres_table, const String & postgres_schema, bool use_nulls, bool with_primary_key, bool with_replica_identity_index, const Strings & columns) { PostgreSQLTableStructure table; @@ -302,6 +303,10 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( ? " AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')" : fmt::format(" AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = {})", quoteString(postgres_schema)); + std::string columns_part; + if (!columns.empty()) + columns_part = fmt::format(" AND attname IN ('{}')", boost::algorithm::join(columns, "','")); + std::string query = fmt::format( "SELECT attname AS name, " /// column name "format_type(atttypid, atttypmod) AS type, " /// data type @@ -312,9 +317,9 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "attnum as att_num, " "attgenerated as generated " /// if column has GENERATED "FROM pg_attribute " - "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) " + "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) {}" "AND NOT attisdropped AND attnum > 0 " - "ORDER BY attnum ASC", where); + "ORDER BY attnum ASC", where, columns_part); auto postgres_table_with_schema = postgres_schema.empty() ? postgres_table : doubleQuoteString(postgres_schema) + '.' + doubleQuoteString(postgres_table); table.physical_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, false); @@ -415,7 +420,7 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( PostgreSQLTableStructure fetchPostgreSQLTableStructure(pqxx::connection & connection, const String & postgres_table, const String & postgres_schema, bool use_nulls) { pqxx::ReadTransaction tx(connection); - auto result = fetchPostgreSQLTableStructure(tx, postgres_table, postgres_schema, use_nulls, false, false); + auto result = fetchPostgreSQLTableStructure(tx, postgres_table, postgres_schema, use_nulls, false, false, {}); tx.commit(); return result; } @@ -433,17 +438,17 @@ std::set fetchPostgreSQLTablesList(pqxx::connection & connection, const template PostgreSQLTableStructure fetchPostgreSQLTableStructure( pqxx::ReadTransaction & tx, const String & postgres_table, const String & postgres_schema, - bool use_nulls, bool with_primary_key, bool with_replica_identity_index); + bool use_nulls, bool with_primary_key, bool with_replica_identity_index, const Strings & columns); template PostgreSQLTableStructure fetchPostgreSQLTableStructure( pqxx::ReplicationTransaction & tx, const String & postgres_table, const String & postgres_schema, - bool use_nulls, bool with_primary_key, bool with_replica_identity_index); + bool use_nulls, bool with_primary_key, bool with_replica_identity_index, const Strings & columns); template PostgreSQLTableStructure fetchPostgreSQLTableStructure( pqxx::nontransaction & tx, const String & postgres_table, const String & postrges_schema, - bool use_nulls, bool with_primary_key, bool with_replica_identity_index); + bool use_nulls, bool with_primary_key, bool with_replica_identity_index, const Strings & columns); std::set fetchPostgreSQLTablesList(pqxx::work & tx, const String & postgres_schema); diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h index 25ece6909fd..6f7bae44c35 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h @@ -48,7 +48,7 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( template PostgreSQLTableStructure fetchPostgreSQLTableStructure( T & tx, const String & postgres_table, const String & postgres_schema, bool use_nulls = true, - bool with_primary_key = false, bool with_replica_identity_index = false); + bool with_primary_key = false, bool with_replica_identity_index = false, const Strings & columns = {}); template std::set fetchPostgreSQLTablesList(T & tx, const String & postgres_schema); diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index 471730fce29..5af9eb1920e 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -220,7 +220,7 @@ void registerDatabaseSQLite(DatabaseFactory & factory) return std::make_shared(args.context, engine_define, args.create_query.attach, database_path); }; - factory.registerDatabase("SQLite", create_fn); + factory.registerDatabase("SQLite", create_fn, {.supports_arguments = true}); } } diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h index f06f5ba8e17..95bb5ac2a11 100644 --- a/src/Disks/DiskEncrypted.h +++ b/src/Disks/DiskEncrypted.h @@ -350,6 +350,12 @@ public: return delegate; } + UInt32 getRefCount(const String & path) const override + { + auto wrapped_path = wrappedPath(path); + return delegate->getRefCount(wrapped_path); + } + #if USE_AWS_S3 std::shared_ptr getS3StorageClient() const override { diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 56bfa019819..0234f8e1dca 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -28,6 +28,7 @@ extern const Event CachedReadBufferReadFromCacheMicroseconds; extern const Event CachedReadBufferCacheWriteMicroseconds; extern const Event CachedReadBufferReadFromSourceBytes; extern const Event CachedReadBufferReadFromCacheBytes; +extern const Event CachedReadBufferPredownloadedBytes; extern const Event CachedReadBufferCacheWriteBytes; extern const Event CachedReadBufferCreateBufferMicroseconds; @@ -644,6 +645,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) size_t current_predownload_size = std::min(current_impl_buffer_size, bytes_to_predownload); ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, current_impl_buffer_size); + ProfileEvents::increment(ProfileEvents::CachedReadBufferPredownloadedBytes, current_impl_buffer_size); std::string failure_reason; bool continue_predownload = file_segment.reserve( diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index a1e7eaf7c3f..913b9e3829a 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -152,6 +152,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects; format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence; format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields; + format_settings.json.empty_as_default = settings.input_format_json_empty_as_default; format_settings.json.type_json_skip_duplicated_paths = settings.type_json_skip_duplicated_paths; format_settings.null_as_default = settings.input_format_null_as_default; format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields; @@ -240,6 +241,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.orc.output_string_as_string = settings.output_format_orc_string_as_string; format_settings.orc.output_compression_method = settings.output_format_orc_compression_method; format_settings.orc.output_row_index_stride = settings.output_format_orc_row_index_stride; + format_settings.orc.output_dictionary_key_size_threshold = settings.output_format_orc_dictionary_key_size_threshold; format_settings.orc.use_fast_decoder = settings.input_format_orc_use_fast_decoder; format_settings.orc.filter_push_down = settings.input_format_orc_filter_push_down; format_settings.orc.reader_time_zone_name = settings.input_format_orc_reader_time_zone_name; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index fa253584090..0c8c4eceb18 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -237,6 +237,7 @@ struct FormatSettings bool infer_incomplete_types_as_strings = true; bool throw_on_bad_escape_sequence = true; bool ignore_unnecessary_fields = true; + bool empty_as_default = false; bool type_json_skip_duplicated_paths = false; } json{}; @@ -415,6 +416,7 @@ struct FormatSettings UInt64 output_row_index_stride = 10'000; String reader_time_zone_name = "GMT"; bool dictionary_as_low_cardinality = true; + double output_dictionary_key_size_threshold = 0.0; } orc{}; /// For capnProto format we should determine how to diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 123f2e4f608..e4d43140ca0 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -2,12 +2,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include @@ -286,11 +288,19 @@ namespace JSONUtils return true; } - if (as_nullable) - return SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(column, in, format_settings, serialization); + auto deserialize = [as_nullable, &format_settings, &serialization](IColumn & column_, ReadBuffer & buf) -> bool + { + if (as_nullable) + return SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(column_, buf, format_settings, serialization); - serialization->deserializeTextJSON(column, in, format_settings); - return true; + serialization->deserializeTextJSON(column_, buf, format_settings); + return true; + }; + + if (format_settings.json.empty_as_default) + return JSONUtils::deserializeEmpyStringAsDefaultOrNested(column, in, deserialize); + else + return deserialize(column, in); } catch (Exception & e) { @@ -920,6 +930,78 @@ namespace JSONUtils } } + template + ReturnType deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested) + { + static constexpr auto throw_exception = std::is_same_v; + + static constexpr auto EMPTY_STRING = "\"\""; + static constexpr auto EMPTY_STRING_LENGTH = std::string_view(EMPTY_STRING).length(); + + if (istr.eof() || *istr.position() != EMPTY_STRING[0]) + return deserialize_nested(column, istr); + + auto do_deserialize = [](IColumn & column_, ReadBuffer & buf, auto && check_for_empty_string, auto && deserialize) -> ReturnType + { + if (check_for_empty_string(buf)) + { + column_.insertDefault(); + return ReturnType(default_column_return_value); + } + return deserialize(column_, buf); + }; + + if (istr.available() >= EMPTY_STRING_LENGTH) + { + /// We have enough data in buffer to check if we have an empty string. + auto check_for_empty_string = [](ReadBuffer & buf) -> bool + { + auto * pos = buf.position(); + if (checkString(EMPTY_STRING, buf)) + return true; + buf.position() = pos; + return false; + }; + + return do_deserialize(column, istr, check_for_empty_string, deserialize_nested); + } + + /// We don't have enough data in buffer to check if we have an empty string. + /// Use PeekableReadBuffer to make a checkpoint before checking for an + /// empty string and rollback if check was failed. + + auto check_for_empty_string = [](ReadBuffer & buf) -> bool + { + auto & peekable_buf = assert_cast(buf); + peekable_buf.setCheckpoint(); + SCOPE_EXIT(peekable_buf.dropCheckpoint()); + if (checkString(EMPTY_STRING, peekable_buf)) + return true; + peekable_buf.rollbackToCheckpoint(); + return false; + }; + + auto deserialize_nested_with_check = [&deserialize_nested](IColumn & column_, ReadBuffer & buf) -> ReturnType + { + auto & peekable_buf = assert_cast(buf); + if constexpr (throw_exception) + deserialize_nested(column_, peekable_buf); + else if (!deserialize_nested(column_, peekable_buf)) + return ReturnType(false); + + if (unlikely(peekable_buf.hasUnreadData())) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect state while parsing JSON: PeekableReadBuffer has unread data in own memory: {}", String(peekable_buf.position(), peekable_buf.available())); + + return ReturnType(true); + }; + + PeekableReadBuffer peekable_buf(istr, true); + return do_deserialize(column, peekable_buf, check_for_empty_string, deserialize_nested_with_check); + } + + template void deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); + template bool deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); + template bool deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); } } diff --git a/src/Formats/JSONUtils.h b/src/Formats/JSONUtils.h index 622703947b9..492da52eb7e 100644 --- a/src/Formats/JSONUtils.h +++ b/src/Formats/JSONUtils.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace DB @@ -146,6 +147,16 @@ namespace JSONUtils bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name, const FormatSettings::JSON & settings); void skipTheRestOfObject(ReadBuffer & in, const FormatSettings::JSON & settings); + + template + using NestedDeserialize = std::function; + + template + ReturnType deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); + + extern template void deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); + extern template bool deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); + extern template bool deserializeEmpyStringAsDefaultOrNested(IColumn & column, ReadBuffer & istr, const NestedDeserialize & deserialize_nested); } } diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h index 40f780d82a8..cd98e0f5875 100644 --- a/src/Functions/FunctionStringOrArrayToT.h +++ b/src/Functions/FunctionStringOrArrayToT.h @@ -27,7 +27,8 @@ class FunctionStringOrArrayToT : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create(ContextPtr) + static FunctionPtr create(ContextPtr) { return createImpl(); } + static FunctionPtr createImpl() { return std::make_shared(); } diff --git a/src/Functions/array/arrayResize.cpp b/src/Functions/array/arrayResize.cpp index 8f4ea69fc5d..fe928f22d38 100644 --- a/src/Functions/array/arrayResize.cpp +++ b/src/Functions/array/arrayResize.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -21,117 +21,99 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -class FunctionArrayResize : public IFunction +DataTypePtr FunctionArrayResize::getReturnTypeImpl(const DataTypes & arguments) const { -public: - static constexpr auto name = "arrayResize"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + const size_t number_of_arguments = arguments.size(); - String getName() const override { return name; } + if (number_of_arguments < 2 || number_of_arguments > 3) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", + getName(), number_of_arguments); - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } + if (arguments[0]->onlyNull()) + return arguments[0]; - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + const auto * array_type = typeid_cast(arguments[0].get()); + if (!array_type) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "First argument for function {} must be an array but it has type {}.", + getName(), arguments[0]->getName()); - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + if (WhichDataType(array_type->getNestedType()).isNothing()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} cannot resize {}", getName(), array_type->getName()); + + if (!isInteger(removeNullable(arguments[1])) && !arguments[1]->onlyNull()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Argument {} for function {} must be integer but it has type {}.", + toString(1), getName(), arguments[1]->getName()); + + if (number_of_arguments == 2) + return arguments[0]; + else /* if (number_of_arguments == 3) */ + return std::make_shared(getLeastSupertype(DataTypes{array_type->getNestedType(), arguments[2]})); +} + +ColumnPtr FunctionArrayResize::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type, size_t input_rows_count) const +{ + if (return_type->onlyNull()) + return return_type->createColumnConstWithDefaultValue(input_rows_count); + + auto result_column = return_type->createColumn(); + + auto array_column = arguments[0].column; + auto size_column = arguments[1].column; + + if (!arguments[0].type->equals(*return_type)) + array_column = castColumn(arguments[0], return_type); + + const DataTypePtr & return_nested_type = typeid_cast(*return_type).getNestedType(); + size_t size = array_column->size(); + + ColumnPtr appended_column; + if (arguments.size() == 3) { - const size_t number_of_arguments = arguments.size(); + appended_column = arguments[2].column; + if (!arguments[2].type->equals(*return_nested_type)) + appended_column = castColumn(arguments[2], return_nested_type); + } + else + appended_column = return_nested_type->createColumnConstWithDefaultValue(size); - if (number_of_arguments < 2 || number_of_arguments > 3) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", - getName(), number_of_arguments); + std::unique_ptr array_source; + std::unique_ptr value_source; - if (arguments[0]->onlyNull()) - return arguments[0]; + bool is_const = false; - const auto * array_type = typeid_cast(arguments[0].get()); - if (!array_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "First argument for function {} must be an array but it has type {}.", - getName(), arguments[0]->getName()); - - if (WhichDataType(array_type->getNestedType()).isNothing()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} cannot resize {}", getName(), array_type->getName()); - - if (!isInteger(removeNullable(arguments[1])) && !arguments[1]->onlyNull()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Argument {} for function {} must be integer but it has type {}.", - toString(1), getName(), arguments[1]->getName()); - - if (number_of_arguments == 2) - return arguments[0]; - else /* if (number_of_arguments == 3) */ - return std::make_shared(getLeastSupertype(DataTypes{array_type->getNestedType(), arguments[2]})); + if (const auto * const_array_column = typeid_cast(array_column.get())) + { + is_const = true; + array_column = const_array_column->getDataColumnPtr(); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type, size_t input_rows_count) const override + if (const auto * argument_column_array = typeid_cast(array_column.get())) + array_source = GatherUtils::createArraySource(*argument_column_array, is_const, size); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "First arguments for function {} must be array.", getName()); + + + bool is_appended_const = false; + if (const auto * const_appended_column = typeid_cast(appended_column.get())) { - if (return_type->onlyNull()) - return return_type->createColumnConstWithDefaultValue(input_rows_count); - - auto result_column = return_type->createColumn(); - - auto array_column = arguments[0].column; - auto size_column = arguments[1].column; - - if (!arguments[0].type->equals(*return_type)) - array_column = castColumn(arguments[0], return_type); - - const DataTypePtr & return_nested_type = typeid_cast(*return_type).getNestedType(); - size_t size = array_column->size(); - - ColumnPtr appended_column; - if (arguments.size() == 3) - { - appended_column = arguments[2].column; - if (!arguments[2].type->equals(*return_nested_type)) - appended_column = castColumn(arguments[2], return_nested_type); - } - else - appended_column = return_nested_type->createColumnConstWithDefaultValue(size); - - std::unique_ptr array_source; - std::unique_ptr value_source; - - bool is_const = false; - - if (const auto * const_array_column = typeid_cast(array_column.get())) - { - is_const = true; - array_column = const_array_column->getDataColumnPtr(); - } - - if (const auto * argument_column_array = typeid_cast(array_column.get())) - array_source = GatherUtils::createArraySource(*argument_column_array, is_const, size); - else - throw Exception(ErrorCodes::LOGICAL_ERROR, "First arguments for function {} must be array.", getName()); - - - bool is_appended_const = false; - if (const auto * const_appended_column = typeid_cast(appended_column.get())) - { - is_appended_const = true; - appended_column = const_appended_column->getDataColumnPtr(); - } - - value_source = GatherUtils::createValueSource(*appended_column, is_appended_const, size); - - auto sink = GatherUtils::createArraySink(typeid_cast(*result_column), size); - - if (isColumnConst(*size_column)) - GatherUtils::resizeConstantSize(*array_source, *value_source, *sink, size_column->getInt(0)); - else - GatherUtils::resizeDynamicSize(*array_source, *value_source, *sink, *size_column); - - return result_column; + is_appended_const = true; + appended_column = const_appended_column->getDataColumnPtr(); } - bool useDefaultImplementationForConstants() const override { return true; } - bool useDefaultImplementationForNulls() const override { return false; } -}; + value_source = GatherUtils::createValueSource(*appended_column, is_appended_const, size); + auto sink = GatherUtils::createArraySink(typeid_cast(*result_column), size); + + if (isColumnConst(*size_column)) + GatherUtils::resizeConstantSize(*array_source, *value_source, *sink, size_column->getInt(0)); + else + GatherUtils::resizeDynamicSize(*array_source, *value_source, *sink, *size_column); + + return result_column; +} REGISTER_FUNCTION(ArrayResize) { diff --git a/src/Functions/array/arrayResize.h b/src/Functions/array/arrayResize.h new file mode 100644 index 00000000000..b6a8ccbbc3c --- /dev/null +++ b/src/Functions/array/arrayResize.h @@ -0,0 +1,28 @@ +#pragma once +#include +#include + +namespace DB +{ + +class FunctionArrayResize : public IFunction +{ +public: + static constexpr auto name = "arrayResize"; + static FunctionPtr createImpl() { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return createImpl(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type, size_t input_rows_count) const override; + bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForNulls() const override { return false; } +}; + +} diff --git a/src/Functions/array/arrayZip.cpp b/src/Functions/array/arrayZip.cpp index 6e1cc0f7788..2f8c9a3af02 100644 --- a/src/Functions/array/arrayZip.cpp +++ b/src/Functions/array/arrayZip.cpp @@ -15,7 +15,6 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int SIZES_OF_ARRAYS_DONT_MATCH; -extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int ILLEGAL_COLUMN; } @@ -38,13 +37,6 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.empty()) - throw Exception( - ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, - "Function {} needs at least one argument; passed {}.", - getName(), - arguments.size()); - DataTypes arguments_types; for (size_t index = 0; index < arguments.size(); ++index) { @@ -68,9 +60,16 @@ public: } ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { size_t num_arguments = arguments.size(); + if (num_arguments == 0) + { + auto res_col = result_type->createColumn(); + res_col->insertDefault(); + return ColumnConst::create(std::move(res_col), input_rows_count); + } + Columns holders(num_arguments); Columns tuple_columns(num_arguments); diff --git a/src/Functions/array/emptyArrayToSingle.cpp b/src/Functions/array/emptyArrayToSingle.cpp index 2071abf9911..5699a4024a1 100644 --- a/src/Functions/array/emptyArrayToSingle.cpp +++ b/src/Functions/array/emptyArrayToSingle.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -20,35 +20,6 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } - -/** emptyArrayToSingle(arr) - replace empty arrays with arrays of one element with a default value. - */ -class FunctionEmptyArrayToSingle : public IFunction -{ -public: - static constexpr auto name = "emptyArrayToSingle"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - - String getName() const override { return name; } - - size_t getNumberOfArguments() const override { return 1; } - bool useDefaultImplementationForConstants() const override { return true; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - const DataTypeArray * array_type = checkAndGetDataType(arguments[0].get()); - if (!array_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument for function {} must be array.", getName()); - - return arguments[0]; - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override; -}; - - namespace { namespace FunctionEmptyArrayToSingleImpl @@ -366,6 +337,14 @@ namespace } } +DataTypePtr FunctionEmptyArrayToSingle::getReturnTypeImpl(const DataTypes & arguments) const +{ + const DataTypeArray * array_type = checkAndGetDataType(arguments[0].get()); + if (!array_type) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument for function {} must be array.", getName()); + + return arguments[0]; +} ColumnPtr FunctionEmptyArrayToSingle::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { diff --git a/src/Functions/array/emptyArrayToSingle.h b/src/Functions/array/emptyArrayToSingle.h new file mode 100644 index 00000000000..f8b67101471 --- /dev/null +++ b/src/Functions/array/emptyArrayToSingle.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include + +namespace DB +{ + +/** emptyArrayToSingle(arr) - replace empty arrays with arrays of one element with a default value. + */ +class FunctionEmptyArrayToSingle : public IFunction +{ +public: + static constexpr auto name = "emptyArrayToSingle"; + static FunctionPtr createImpl() { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return createImpl(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override; + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override; +}; + +} diff --git a/src/Functions/array/length.cpp b/src/Functions/array/length.cpp index 760506194fa..949a5441e58 100644 --- a/src/Functions/array/length.cpp +++ b/src/Functions/array/length.cpp @@ -1,65 +1,7 @@ -#include -#include -#include - +#include namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - -/** Calculates the length of a string in bytes. - */ -struct LengthImpl -{ - static constexpr auto is_fixed_to_constant = true; - - static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray & res, size_t input_rows_count) - { - for (size_t i = 0; i < input_rows_count; ++i) - res[i] = offsets[i] - 1 - offsets[i - 1]; - } - - static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t n, UInt64 & res, size_t) - { - res = n; - } - - static void vectorFixedToVector(const ColumnString::Chars & /*data*/, size_t /*n*/, PaddedPODArray & /*res*/, size_t) - { - } - - static void array(const ColumnString::Offsets & offsets, PaddedPODArray & res, size_t input_rows_count) - { - for (size_t i = 0; i < input_rows_count; ++i) - res[i] = offsets[i] - offsets[i - 1]; - } - - [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray &, size_t) - { - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to UUID argument"); - } - - [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray &, size_t) - { - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv6 argument"); - } - - [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray &, size_t) - { - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv4 argument"); - } -}; - - -struct NameLength -{ - static constexpr auto name = "length"; -}; - -using FunctionLength = FunctionStringOrArrayToT; REGISTER_FUNCTION(Length) { diff --git a/src/Functions/array/length.h b/src/Functions/array/length.h new file mode 100644 index 00000000000..2ecab76e0f8 --- /dev/null +++ b/src/Functions/array/length.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +/** Calculates the length of a string in bytes. + */ +struct LengthImpl +{ + static constexpr auto is_fixed_to_constant = true; + + static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray & res, size_t input_rows_count) + { + for (size_t i = 0; i < input_rows_count; ++i) + res[i] = offsets[i] - 1 - offsets[i - 1]; + } + + static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t n, UInt64 & res, size_t) + { + res = n; + } + + static void vectorFixedToVector(const ColumnString::Chars & /*data*/, size_t /*n*/, PaddedPODArray & /*res*/, size_t) + { + } + + static void array(const ColumnString::Offsets & offsets, PaddedPODArray & res, size_t input_rows_count) + { + for (size_t i = 0; i < input_rows_count; ++i) + res[i] = offsets[i] - offsets[i - 1]; + } + + [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray &, size_t) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to UUID argument"); + } + + [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray &, size_t) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv6 argument"); + } + + [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray &, size_t) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv4 argument"); + } +}; + + +struct NameLength +{ + static constexpr auto name = "length"; +}; + +using FunctionLength = FunctionStringOrArrayToT; + +} diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp index 3cf3efaf534..afacf164a00 100644 --- a/src/Functions/printf.cpp +++ b/src/Functions/printf.cpp @@ -50,13 +50,6 @@ private: return executeNonconstant(input); } - [[maybe_unused]] String toString() const - { - WriteBufferFromOwnString buf; - buf << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure() << "\n"; - return buf.str(); - } - private: ColumnWithTypeAndName executeLiteral(std::string_view literal) const { @@ -231,9 +224,7 @@ public: const auto & instruction = instructions[i]; try { - // std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl; concat_args[i] = instruction.execute(); - // std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl; } catch (const fmt::v9::format_error & e) { @@ -358,7 +349,14 @@ private: REGISTER_FUNCTION(Printf) { - factory.registerFunction(); + factory.registerFunction( + FunctionDocumentation{.description=R"( +The `printf` function formats the given string with the values (strings, integers, floating-points etc.) listed in the arguments, similar to printf function in C++. +The format string can contain format specifiers starting with `%` character. +Anything not contained in `%` and the following format specifier is considered literal text and copied verbatim into the output. +Literal `%` character can be escaped by `%%`.)", .examples{{"sum", "select printf('%%%s %s %d', 'Hello', 'World', 2024);", "%Hello World 2024"}}, .categories{"String"} +}); + } } diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 855aa36b159..81600b2b6eb 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -33,15 +34,18 @@ namespace void updateUserFromQueryImpl( User & user, const ASTCreateUserQuery & query, - const std::optional auth_data, + const std::vector authentication_methods, const std::shared_ptr & override_name, const std::optional & override_default_roles, const std::optional & override_settings, const std::optional & override_grantees, const std::optional & valid_until, + bool reset_authentication_methods, + bool replace_authentication_methods, bool allow_implicit_no_password, bool allow_no_password, - bool allow_plaintext_password) + bool allow_plaintext_password, + std::size_t max_number_of_authentication_methods) { if (override_name) user.setName(override_name->toString()); @@ -50,25 +54,77 @@ namespace else if (query.names->size() == 1) user.setName(query.names->front()->toString()); - if (!query.attach && !query.alter && !auth_data && !allow_implicit_no_password) + if (!query.attach && !query.alter && authentication_methods.empty() && !allow_implicit_no_password) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication type NO_PASSWORD must " "be explicitly specified, check the setting allow_implicit_no_password " "in the server configuration"); - if (auth_data) - user.auth_data = *auth_data; - - if (auth_data || !query.alter) + // if user does not have an authentication method and it has not been specified in the query, + // add a default one + if (user.authentication_methods.empty() && authentication_methods.empty()) { - auto auth_type = user.auth_data.getType(); - if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || - ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + user.authentication_methods.emplace_back(); + } + + // 1. an IDENTIFIED WITH will drop existing authentication methods in favor of new ones. + if (replace_authentication_methods) + { + user.authentication_methods.clear(); + } + + // drop existing ones and keep the most recent + if (reset_authentication_methods) + { + auto backup_authentication_method = user.authentication_methods.back(); + user.authentication_methods.clear(); + user.authentication_methods.emplace_back(backup_authentication_method); + } + + // max_number_of_authentication_methods == 0 means unlimited + if (!authentication_methods.empty() && max_number_of_authentication_methods != 0) + { + // we only check if user exceeds the allowed quantity of authentication methods in case the create/alter query includes + // authentication information. Otherwise, we can bypass this check to avoid blocking non-authentication related alters. + auto number_of_authentication_methods = user.authentication_methods.size() + authentication_methods.size(); + if (number_of_authentication_methods > max_number_of_authentication_methods) { throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", - toString(auth_type), - AuthenticationTypeInfo::get(auth_type).name); + "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user. " + "Check the `max_authentication_methods_per_user` setting"); + } + } + + for (const auto & authentication_method : authentication_methods) + { + user.authentication_methods.emplace_back(authentication_method); + } + + bool has_no_password_authentication_method = std::find_if(user.authentication_methods.begin(), + user.authentication_methods.end(), + [](const AuthenticationData & authentication_data) + { + return authentication_data.getType() == AuthenticationType::NO_PASSWORD; + }) != user.authentication_methods.end(); + + if (has_no_password_authentication_method && user.authentication_methods.size() > 1) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods"); + } + + if (!query.alter) + { + for (const auto & authentication_method : user.authentication_methods) + { + auto auth_type = authentication_method.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", + toString(auth_type), + AuthenticationTypeInfo::get(auth_type).name); + } } } @@ -156,9 +212,14 @@ BlockIO InterpreterCreateUserQuery::execute() bool no_password_allowed = access_control.isNoPasswordAllowed(); bool plaintext_password_allowed = access_control.isPlaintextPasswordAllowed(); - std::optional auth_data; - if (query.auth_data) - auth_data = AuthenticationData::fromAST(*query.auth_data, getContext(), !query.attach); + std::vector authentication_methods; + if (!query.authentication_methods.empty()) + { + for (const auto & authentication_method_ast : query.authentication_methods) + { + authentication_methods.push_back(AuthenticationData::fromAST(*authentication_method_ast, getContext(), !query.attach)); + } + } std::optional valid_until; if (query.valid_until) @@ -207,8 +268,10 @@ BlockIO InterpreterCreateUserQuery::execute() { auto updated_user = typeid_cast>(entity->clone()); updateUserFromQueryImpl( - *updated_user, query, auth_data, {}, default_roles_from_query, settings_from_query, grantees_from_query, - valid_until, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + *updated_user, query, authentication_methods, {}, default_roles_from_query, settings_from_query, grantees_from_query, + valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, + implicit_no_password_allowed, no_password_allowed, + plaintext_password_allowed, getContext()->getServerSettings().max_authentication_methods_per_user); return updated_user; }; @@ -227,8 +290,10 @@ BlockIO InterpreterCreateUserQuery::execute() { auto new_user = std::make_shared(); updateUserFromQueryImpl( - *new_user, query, auth_data, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, - valid_until, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + *new_user, query, authentication_methods, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, + valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, + implicit_no_password_allowed, no_password_allowed, + plaintext_password_allowed, getContext()->getServerSettings().max_authentication_methods_per_user); new_users.emplace_back(std::move(new_user)); } @@ -265,17 +330,41 @@ BlockIO InterpreterCreateUserQuery::execute() } -void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password) +void InterpreterCreateUserQuery::updateUserFromQuery( + User & user, + const ASTCreateUserQuery & query, + bool allow_no_password, + bool allow_plaintext_password, + std::size_t max_number_of_authentication_methods) { - std::optional auth_data; - if (query.auth_data) - auth_data = AuthenticationData::fromAST(*query.auth_data, {}, !query.attach); + std::vector authentication_methods; + if (!query.authentication_methods.empty()) + { + for (const auto & authentication_method_ast : query.authentication_methods) + { + authentication_methods.emplace_back(AuthenticationData::fromAST(*authentication_method_ast, {}, !query.attach)); + } + } std::optional valid_until; if (query.valid_until) valid_until = getValidUntilFromAST(query.valid_until, {}); - updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, valid_until, allow_no_password, allow_plaintext_password, true); + updateUserFromQueryImpl( + user, + query, + authentication_methods, + {}, + {}, + {}, + {}, + valid_until, + query.reset_authentication_methods_to_new, + query.replace_authentication_methods, + allow_no_password, + allow_plaintext_password, + true, + max_number_of_authentication_methods); } void registerInterpreterCreateUserQuery(InterpreterFactory & factory) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.h b/src/Interpreters/Access/InterpreterCreateUserQuery.h index 372066cfd5e..fea87d33703 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.h +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.h @@ -17,7 +17,12 @@ public: BlockIO execute() override; - static void updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password); + static void updateUserFromQuery( + User & user, + const ASTCreateUserQuery & query, + bool allow_no_password, + bool allow_plaintext_password, + std::size_t max_number_of_authentication_methods); private: ASTPtr query_ptr; diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 96d8e55a74c..ef6ddf1866d 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -64,8 +64,10 @@ namespace query->default_roles = user.default_roles.toASTWithNames(*access_control); } - if (user.auth_data.getType() != AuthenticationType::NO_PASSWORD) - query->auth_data = user.auth_data.toAST(); + for (const auto & authentication_method : user.authentication_methods) + { + query->authentication_methods.push_back(authentication_method.toAST()); + } if (user.valid_until) { diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 9ebc73260b6..44862344df0 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -2028,8 +2028,9 @@ ActionsDAG::SplitResult ActionsDAG::split(std::unordered_set split return {std::move(first_actions), std::move(second_actions), std::move(split_nodes_mapping)}; } -ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & array_joined_columns) const +ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const Names & array_joined_columns) const { + std::unordered_set array_joined_columns_set(array_joined_columns.begin(), array_joined_columns.end()); struct Frame { const Node * node = nullptr; @@ -2072,7 +2073,7 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const NameSet & if (cur.next_child_to_visit == cur.node->children.size()) { bool depend_on_array_join = false; - if (cur.node->type == ActionType::INPUT && array_joined_columns.contains(cur.node->result_name)) + if (cur.node->type == ActionType::INPUT && array_joined_columns_set.contains(cur.node->result_name)) depend_on_array_join = true; for (const auto * child : cur.node->children) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 2459878ce20..0d6b1ce0e04 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -340,7 +340,7 @@ public: SplitResult split(std::unordered_set split_nodes, bool create_split_nodes_mapping = false, bool avoid_duplicate_inputs = false) const; /// Splits actions into two parts. Returned first half may be swapped with ARRAY JOIN. - SplitResult splitActionsBeforeArrayJoin(const NameSet & array_joined_columns) const; + SplitResult splitActionsBeforeArrayJoin(const Names & array_joined_columns) const; /// Splits actions into two parts. First part has minimal size sufficient for calculation of column_name. /// Outputs of initial actions must contain column_name. diff --git a/src/Interpreters/ArrayJoin.h b/src/Interpreters/ArrayJoin.h new file mode 100644 index 00000000000..6a2bbef79db --- /dev/null +++ b/src/Interpreters/ArrayJoin.h @@ -0,0 +1,13 @@ +#pragma once +#include + +namespace DB +{ + +struct ArrayJoin +{ + Names columns; + bool is_left = false; +}; + +} diff --git a/src/Interpreters/ArrayJoinAction.cpp b/src/Interpreters/ArrayJoinAction.cpp index df7a0b48057..f49b3af9c58 100644 --- a/src/Interpreters/ArrayJoinAction.cpp +++ b/src/Interpreters/ArrayJoinAction.cpp @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include #include @@ -59,26 +62,31 @@ ColumnWithTypeAndName convertArrayJoinColumn(const ColumnWithTypeAndName & src_c return array_col; } -ArrayJoinAction::ArrayJoinAction(const NameSet & array_joined_columns_, bool array_join_is_left, ContextPtr context) - : columns(array_joined_columns_) - , is_left(array_join_is_left) - , is_unaligned(context->getSettingsRef().enable_unaligned_array_join) - , max_block_size(context->getSettingsRef().max_block_size) +ArrayJoinAction::ArrayJoinAction(const Names & columns_, bool is_left_, bool is_unaligned_, size_t max_block_size_) + : columns(columns_.begin(), columns_.end()) + , is_left(is_left_) + , is_unaligned(is_unaligned_) + , max_block_size(max_block_size_) { if (columns.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "No arrays to join"); if (is_unaligned) { - function_length = FunctionFactory::instance().get("length", context); - function_greatest = FunctionFactory::instance().get("greatest", context); - function_array_resize = FunctionFactory::instance().get("arrayResize", context); + function_length = std::make_unique(FunctionLength::createImpl()); + function_array_resize = std::make_unique(FunctionArrayResize::createImpl()); } else if (is_left) - function_builder = FunctionFactory::instance().get("emptyArrayToSingle", context); + function_builder = std::make_unique(FunctionEmptyArrayToSingle::createImpl()); } -void ArrayJoinAction::prepare(ColumnsWithTypeAndName & sample) const +void ArrayJoinAction::prepare(const Names & columns, ColumnsWithTypeAndName & sample) +{ + NameSet columns_set(columns.begin(), columns.end()); + prepare(columns_set, sample); +} + +void ArrayJoinAction::prepare(const NameSet & columns, ColumnsWithTypeAndName & sample) { for (auto & current : sample) { @@ -103,6 +111,35 @@ ArrayJoinResultIteratorPtr ArrayJoinAction::execute(Block block) return std::make_unique(this, std::move(block)); } +static void updateMaxLength(ColumnUInt64 & max_length, UInt64 length) +{ + for (auto & value : max_length.getData()) + value = std::max(value, length); +} + +static void updateMaxLength(ColumnUInt64 & max_length, const IColumn & length) +{ + if (const auto * length_const = typeid_cast(&length)) + { + updateMaxLength(max_length, length_const->getUInt(0)); + return; + } + + const auto * length_uint64 = typeid_cast(&length); + if (!length_uint64) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected UInt64 for array length, got {}", length.getName()); + + auto & max_lenght_data = max_length.getData(); + const auto & length_data = length_uint64->getData(); + size_t num_rows = max_lenght_data.size(); + if (num_rows != length_data.size()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Different columns sizes in ARRAY JOIN: {} and {}", num_rows, length_data.size()); + + for (size_t row = 0; row < num_rows; ++row) + max_lenght_data[row] = std::max(max_lenght_data[row], length_data[row]); +} ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_join_, Block block_) : array_join(array_join_), block(std::move(block_)), total_rows(block.rows()), current_row(0) @@ -111,7 +148,6 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j bool is_unaligned = array_join->is_unaligned; bool is_left = array_join->is_left; const auto & function_length = array_join->function_length; - const auto & function_greatest = array_join->function_greatest; const auto & function_array_resize = array_join->function_array_resize; const auto & function_builder = array_join->function_builder; @@ -125,11 +161,7 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j /// Resize all array joined columns to the longest one, (at least 1 if LEFT ARRAY JOIN), padded with default values. auto rows = block.rows(); auto uint64 = std::make_shared(); - ColumnWithTypeAndName column_of_max_length{{}, uint64, {}}; - if (is_left) - column_of_max_length = ColumnWithTypeAndName(uint64->createColumnConst(rows, 1u), uint64, {}); - else - column_of_max_length = ColumnWithTypeAndName(uint64->createColumnConst(rows, 0u), uint64, {}); + auto max_length = ColumnUInt64::create(rows, (is_left ? 1u : 0u)); for (const auto & name : columns) { @@ -138,11 +170,10 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j ColumnWithTypeAndName array_col = convertArrayJoinColumn(src_col); ColumnsWithTypeAndName tmp_block{array_col}; //, {{}, uint64, {}}}; auto len_col = function_length->build(tmp_block)->execute(tmp_block, uint64, rows); - - ColumnsWithTypeAndName tmp_block2{column_of_max_length, {len_col, uint64, {}}}; - column_of_max_length.column = function_greatest->build(tmp_block2)->execute(tmp_block2, uint64, rows); + updateMaxLength(*max_length, *len_col); } + ColumnWithTypeAndName column_of_max_length{std::move(max_length), uint64, {}}; for (const auto & name : columns) { auto & src_col = block.getByName(name); diff --git a/src/Interpreters/ArrayJoinAction.h b/src/Interpreters/ArrayJoinAction.h index 603f22ef245..b76822e6b71 100644 --- a/src/Interpreters/ArrayJoinAction.h +++ b/src/Interpreters/ArrayJoinAction.h @@ -33,14 +33,14 @@ public: /// For unaligned [LEFT] ARRAY JOIN FunctionOverloadResolverPtr function_length; - FunctionOverloadResolverPtr function_greatest; FunctionOverloadResolverPtr function_array_resize; /// For LEFT ARRAY JOIN. FunctionOverloadResolverPtr function_builder; - ArrayJoinAction(const NameSet & array_joined_columns_, bool array_join_is_left, ContextPtr context); - void prepare(ColumnsWithTypeAndName & sample) const; + ArrayJoinAction(const Names & columns_, bool is_left_, bool is_unaligned_, size_t max_block_size_); + static void prepare(const NameSet & columns, ColumnsWithTypeAndName & sample); + static void prepare(const Names & columns, ColumnsWithTypeAndName & sample); ArrayJoinResultIteratorPtr execute(Block block); }; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 60db406ca72..ffe9a611014 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -718,7 +718,12 @@ FileCache::getOrSet( } } - chassert(file_segments_limit ? file_segments.back()->range().left <= result_range.right : file_segments.back()->range().contains(result_range.right)); + chassert(file_segments_limit + ? file_segments.back()->range().left <= result_range.right + : file_segments.back()->range().contains(result_range.right), + fmt::format("Unexpected state. Back: {}, result range: {}, limit: {}", + file_segments.back()->range().toString(), result_range.toString(), file_segments_limit)); + chassert(!file_segments_limit || file_segments.size() <= file_segments_limit); return std::make_unique(std::move(file_segments)); diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 771c6a89caa..1f854c41873 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -532,7 +532,7 @@ void executeQueryWithParallelReplicas( max_replicas_to_use = shard.getAllNodeCount(); } - auto coordinator = std::make_shared(max_replicas_to_use, settings.parallel_replicas_mark_segment_size); + auto coordinator = std::make_shared(max_replicas_to_use); auto external_tables = new_context->getExternalTables(); diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index ac940c62a1a..d906540d6df 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -85,7 +85,9 @@ ConcurrentHashJoin::ConcurrentHashJoin( CurrentMetrics::ConcurrentHashJoinPoolThreads, CurrentMetrics::ConcurrentHashJoinPoolThreadsActive, CurrentMetrics::ConcurrentHashJoinPoolThreadsScheduled, - slots)) + /*max_threads_*/ slots, + /*max_free_threads_*/ 0, + /*queue_size_*/ slots)) , stats_collecting_params(stats_collecting_params_) { hash_joins.resize(slots); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7adfb42fb51..311fd094706 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -121,7 +122,6 @@ #include #include - namespace fs = std::filesystem; namespace ProfileEvents @@ -164,6 +164,9 @@ namespace CurrentMetrics extern const Metric TablesLoaderForegroundThreadsActive; extern const Metric TablesLoaderForegroundThreadsScheduled; extern const Metric IOWriterThreadsScheduled; + extern const Metric BuildVectorSimilarityIndexThreads; + extern const Metric BuildVectorSimilarityIndexThreadsActive; + extern const Metric BuildVectorSimilarityIndexThreadsScheduled; extern const Metric AttachedTable; extern const Metric AttachedView; extern const Metric AttachedDictionary; @@ -297,6 +300,8 @@ struct ContextSharedPart : boost::noncopyable mutable std::unique_ptr load_marks_threadpool; /// Threadpool for loading marks cache. mutable OnceFlag prefetch_threadpool_initialized; mutable std::unique_ptr prefetch_threadpool; /// Threadpool for loading marks cache. + mutable OnceFlag build_vector_similarity_index_threadpool_initialized; + mutable std::unique_ptr build_vector_similarity_index_threadpool; /// Threadpool for vector-similarity index creation. mutable UncompressedCachePtr index_uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks for MergeTree indices. mutable QueryCachePtr query_cache TSA_GUARDED_BY(mutex); /// Cache of query results. mutable MarkCachePtr index_mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files of MergeTree indices. @@ -3297,6 +3302,21 @@ size_t Context::getPrefetchThreadpoolSize() const return config.getUInt(".prefetch_threadpool_pool_size", 100); } +ThreadPool & Context::getBuildVectorSimilarityIndexThreadPool() const +{ + callOnce(shared->build_vector_similarity_index_threadpool_initialized, [&] { + size_t pool_size = shared->server_settings.max_build_vector_similarity_index_thread_pool_size > 0 + ? shared->server_settings.max_build_vector_similarity_index_thread_pool_size + : getNumberOfPhysicalCPUCores(); + shared->build_vector_similarity_index_threadpool = std::make_unique( + CurrentMetrics::BuildVectorSimilarityIndexThreads, + CurrentMetrics::BuildVectorSimilarityIndexThreadsActive, + CurrentMetrics::BuildVectorSimilarityIndexThreadsScheduled, + pool_size); + }); + return *shared->build_vector_similarity_index_threadpool; +} + BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const { callOnce(shared->buffer_flush_schedule_pool_initialized, [&] { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 858b4a78430..0daef2243aa 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1097,6 +1097,8 @@ public: /// and make a prefetch by putting a read task to threadpoolReader. size_t getPrefetchThreadpoolSize() const; + ThreadPool & getBuildVectorSimilarityIndexThreadPool() const; + /// Settings for MergeTree background tasks stored in config.xml BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const; BackgroundTaskSchedulingSettings getBackgroundMoveTaskSchedulingSettings() const; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 4c313b3c9a8..edf419d404e 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -1059,16 +1059,16 @@ std::string ExpressionActionsChain::dumpChain() const return ss.str(); } -ExpressionActionsChain::ArrayJoinStep::ArrayJoinStep(ArrayJoinActionPtr array_join_, ColumnsWithTypeAndName required_columns_) +ExpressionActionsChain::ArrayJoinStep::ArrayJoinStep(const Names & array_join_columns_, ColumnsWithTypeAndName required_columns_) : Step({}) - , array_join(std::move(array_join_)) + , array_join_columns(array_join_columns_.begin(), array_join_columns_.end()) , result_columns(std::move(required_columns_)) { for (auto & column : result_columns) { required_columns.emplace_back(NameAndTypePair(column.name, column.type)); - if (array_join->columns.contains(column.name)) + if (array_join_columns.contains(column.name)) { const auto & array = getArrayJoinDataType(column.type); column.type = array->getNestedType(); @@ -1085,12 +1085,12 @@ void ExpressionActionsChain::ArrayJoinStep::finalize(const NameSet & required_ou for (const auto & column : result_columns) { - if (array_join->columns.contains(column.name) || required_output_.contains(column.name)) + if (array_join_columns.contains(column.name) || required_output_.contains(column.name)) new_result_columns.emplace_back(column); } for (const auto & column : required_columns) { - if (array_join->columns.contains(column.name) || required_output_.contains(column.name)) + if (array_join_columns.contains(column.name) || required_output_.contains(column.name)) new_required_columns.emplace_back(column); } diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 7652fe49eab..539c7c8d141 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -22,9 +23,6 @@ class TableJoin; class IJoin; using JoinPtr = std::shared_ptr; -class ArrayJoinAction; -using ArrayJoinActionPtr = std::shared_ptr; - class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; @@ -223,11 +221,11 @@ struct ExpressionActionsChain : WithContext struct ArrayJoinStep : public Step { - ArrayJoinActionPtr array_join; + const NameSet array_join_columns; NamesAndTypesList required_columns; ColumnsWithTypeAndName result_columns; - ArrayJoinStep(ArrayJoinActionPtr array_join_, ColumnsWithTypeAndName required_columns_); + ArrayJoinStep(const Names & array_join_columns_, ColumnsWithTypeAndName required_columns_); NamesAndTypesList getRequiredColumns() const override { return required_columns; } ColumnsWithTypeAndName getResultColumns() const override { return result_columns; } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 7063b2162a0..1235ebb0871 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -215,7 +215,7 @@ NamesAndTypesList ExpressionAnalyzer::getColumnsAfterArrayJoin(ActionsDAG & acti auto array_join = addMultipleArrayJoinAction(actions, is_array_join_left); auto sample_columns = actions.getResultColumns(); - array_join->prepare(sample_columns); + ArrayJoinAction::prepare(array_join.columns, sample_columns); actions = ActionsDAG(sample_columns); NamesAndTypesList new_columns_after_array_join; @@ -889,9 +889,11 @@ const ASTSelectQuery * SelectQueryExpressionAnalyzer::getAggregatingQuery() cons } /// "Big" ARRAY JOIN. -ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAG & actions, bool array_join_is_left) const +ArrayJoin ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAG & actions, bool array_join_is_left) const { - NameSet result_columns; + Names result_columns; + result_columns.reserve(syntax->array_join_result_to_source.size()); + for (const auto & result_source : syntax->array_join_result_to_source) { /// Assign new names to columns, if needed. @@ -902,19 +904,19 @@ ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAG & a } /// Make ARRAY JOIN (replace arrays with their insides) for the columns in these new names. - result_columns.insert(result_source.first); + result_columns.push_back(result_source.first); } - return std::make_shared(result_columns, array_join_is_left, getContext()); + return {std::move(result_columns), array_join_is_left}; } -ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & before_array_join, bool only_types) +std::optional SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & before_array_join, bool only_types) { const auto * select_query = getSelectQuery(); auto [array_join_expression_list, is_array_join_left] = select_query->arrayJoinExpressionList(); if (!array_join_expression_list) - return nullptr; + return {}; ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns()); @@ -923,7 +925,7 @@ ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActi auto array_join = addMultipleArrayJoinAction(step.actions()->dag, is_array_join_left); before_array_join = chain.getLastActions(); - chain.steps.push_back(std::make_unique(array_join, step.getResultColumns())); + chain.steps.push_back(std::make_unique(array_join.columns, step.getResultColumns())); chain.addStep(); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index dc038e10594..483be5cc4c4 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -174,7 +174,7 @@ protected: /// Find global subqueries in the GLOBAL IN/JOIN sections. Fills in external_tables. void initGlobalSubqueriesAndExternalTables(bool do_global, bool is_explain); - ArrayJoinActionPtr addMultipleArrayJoinAction(ActionsDAG & actions, bool is_left) const; + ArrayJoin addMultipleArrayJoinAction(ActionsDAG & actions, bool is_left) const; void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions, bool only_consts = false); @@ -234,7 +234,7 @@ struct ExpressionAnalysisResult bool use_grouping_set_key = false; ActionsAndProjectInputsFlagPtr before_array_join; - ArrayJoinActionPtr array_join; + std::optional array_join; ActionsAndProjectInputsFlagPtr before_join; ActionsAndProjectInputsFlagPtr converting_join_columns; JoinPtr join; @@ -388,7 +388,7 @@ private: */ /// Before aggregation: - ArrayJoinActionPtr appendArrayJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & before_array_join, bool only_types); + std::optional appendArrayJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & before_array_join, bool only_types); bool appendJoinLeftKeys(ExpressionActionsChain & chain, bool only_types); JoinPtr appendJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & converting_join_columns); diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index 1b8b45b94ea..230e4cd9691 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -1236,6 +1236,7 @@ IBlocksStreamPtr HashJoin::getNonJoinedBlocks(const Block & left_sample_block, void HashJoin::reuseJoinedData(const HashJoin & join) { + have_compressed = join.have_compressed; data = join.data; from_storage_join = true; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index c1f9b4637f8..60d4abd0ef8 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -968,6 +968,11 @@ void InterpreterCreateQuery::validateMaterializedViewColumnsAndEngine(const ASTC if (database && database->getEngineName() != "Atomic") throw Exception(ErrorCodes::INCORRECT_QUERY, "Refreshable materialized views (except with APPEND) only support Atomic database engine, but database {} has engine {}", create.getDatabase(), database->getEngineName()); + + std::string message; + if (!supportsAtomicRename(&message)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Can't create refreshable materialized view because exchanging files is not supported by the OS ({})", message); } Block input_block; @@ -1060,6 +1065,11 @@ namespace void setNullTableEngine(ASTStorage & storage) { + storage.forEachPointerToChild([](void ** ptr) mutable + { + *ptr = nullptr; + }); + auto engine_ast = std::make_shared(); engine_ast->name = "Null"; engine_ast->no_empty_args = true; @@ -1146,7 +1156,9 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null) { if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE) + { setNullTableEngine(*create.storage); + } } return; } diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index ef560ec3405..5161fd19d87 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -380,100 +380,99 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait) { + if (query.kind != ASTDropQuery::Kind::Detach && query.kind != ASTDropQuery::Kind::Drop && query.kind != ASTDropQuery::Kind::Truncate) + return {}; + const auto & database_name = query.getDatabase(); auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); database = tryGetDatabase(database_name, query.if_exists); - if (database) + if (!database) + return {}; + + bool drop = query.kind == ASTDropQuery::Kind::Drop; + bool truncate = query.kind == ASTDropQuery::Kind::Truncate; + + getContext()->checkAccess(AccessType::DROP_DATABASE, database_name); + + if (query.kind == ASTDropQuery::Kind::Detach && query.permanently) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DETACH PERMANENTLY is not implemented for databases"); + + if (query.if_empty) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DROP IF EMPTY is not implemented for databases"); + + if (!truncate && database->hasReplicationThread()) + database->stopReplication(); + + if (database->shouldBeEmptyOnDetach()) { - if (query.kind == ASTDropQuery::Kind::Detach || query.kind == ASTDropQuery::Kind::Drop - || query.kind == ASTDropQuery::Kind::Truncate) + /// Cancel restarting replicas in that database, wait for remaining RESTART queries to finish. + /// So it will not startup tables concurrently with the flushAndPrepareForShutdown call below. + auto restart_replica_lock = DatabaseCatalog::instance().getLockForDropDatabase(database_name); + + ASTDropQuery query_for_table; + query_for_table.kind = query.kind; + // For truncate operation on database, drop the tables + if (truncate) + query_for_table.kind = query.has_all_tables ? ASTDropQuery::Kind::Truncate : ASTDropQuery::Kind::Drop; + query_for_table.if_exists = true; + query_for_table.if_empty = false; + query_for_table.setDatabase(database_name); + query_for_table.sync = query.sync; + + /// Flush should not be done if shouldBeEmptyOnDetach() == false, + /// since in this case getTablesIterator() may do some additional work, + /// see DatabaseMaterializedMySQL::getTablesIterator() + auto table_context = Context::createCopy(getContext()); + table_context->setInternalQuery(true); + /// Do not hold extra shared pointers to tables + std::vector> tables_to_drop; + // NOTE: This means we wait for all tables to be loaded inside getTablesIterator() call in case of `async_load_databases = true`. + for (auto iterator = database->getTablesIterator(table_context); iterator->isValid(); iterator->next()) { - bool drop = query.kind == ASTDropQuery::Kind::Drop; - bool truncate = query.kind == ASTDropQuery::Kind::Truncate; + auto table_ptr = iterator->table(); + tables_to_drop.push_back({table_ptr->getStorageID(), table_ptr->isDictionary()}); + } - getContext()->checkAccess(AccessType::DROP_DATABASE, database_name); - - if (query.kind == ASTDropQuery::Kind::Detach && query.permanently) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DETACH PERMANENTLY is not implemented for databases"); - - if (query.if_empty) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DROP IF EMPTY is not implemented for databases"); - - if (!truncate && database->hasReplicationThread()) - database->stopReplication(); - - if (database->shouldBeEmptyOnDetach()) + /// Prepare tables for shutdown in parallel. + ThreadPoolCallbackRunnerLocal runner(getDatabaseCatalogDropTablesThreadPool().get(), "DropTables"); + for (const auto & [name, _] : tables_to_drop) + { + auto table_ptr = DatabaseCatalog::instance().getTable(name, table_context); + runner([my_table_ptr = std::move(table_ptr)]() { - /// Cancel restarting replicas in that database, wait for remaining RESTART queries to finish. - /// So it will not startup tables concurrently with the flushAndPrepareForShutdown call below. - auto restart_replica_lock = DatabaseCatalog::instance().getLockForDropDatabase(database_name); + my_table_ptr->flushAndPrepareForShutdown(); + }); + } + runner.waitForAllToFinishAndRethrowFirstError(); - ASTDropQuery query_for_table; - query_for_table.kind = query.kind; - // For truncate operation on database, drop the tables - if (truncate) - query_for_table.kind = query.has_all_tables ? ASTDropQuery::Kind::Truncate : ASTDropQuery::Kind::Drop; - query_for_table.if_exists = true; - query_for_table.if_empty = false; - query_for_table.setDatabase(database_name); - query_for_table.sync = query.sync; - - /// Flush should not be done if shouldBeEmptyOnDetach() == false, - /// since in this case getTablesIterator() may do some additional work, - /// see DatabaseMaterializedMySQL::getTablesIterator() - auto table_context = Context::createCopy(getContext()); - table_context->setInternalQuery(true); - /// Do not hold extra shared pointers to tables - std::vector> tables_to_drop; - // NOTE: This means we wait for all tables to be loaded inside getTablesIterator() call in case of `async_load_databases = true`. - for (auto iterator = database->getTablesIterator(table_context); iterator->isValid(); iterator->next()) - { - auto table_ptr = iterator->table(); - tables_to_drop.push_back({table_ptr->getStorageID(), table_ptr->isDictionary()}); - } - - /// Prepare tables for shutdown in parallel. - ThreadPoolCallbackRunnerLocal runner(getDatabaseCatalogDropTablesThreadPool().get(), "DropTables"); - for (const auto & [name, _] : tables_to_drop) - { - auto table_ptr = DatabaseCatalog::instance().getTable(name, table_context); - runner([my_table_ptr = std::move(table_ptr)]() - { - my_table_ptr->flushAndPrepareForShutdown(); - }); - } - runner.waitForAllToFinishAndRethrowFirstError(); - - for (const auto & table : tables_to_drop) - { - query_for_table.setTable(table.first.getTableName()); - query_for_table.is_dictionary = table.second; - DatabasePtr db; - UUID table_to_wait = UUIDHelpers::Nil; - executeToTableImpl(table_context, query_for_table, db, table_to_wait); - uuids_to_wait.push_back(table_to_wait); - } - } - // only if operation is DETACH - if ((!drop || !truncate) && query.sync) - { - /// Avoid "some tables are still in use" when sync mode is enabled - for (const auto & table_uuid : uuids_to_wait) - database->waitDetachedTableNotInUse(table_uuid); - } - - /// Protects from concurrent CREATE TABLE queries - auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name); - // only if operation is DETACH - if (!drop || !truncate) - database->assertCanBeDetached(true); - - /// DETACH or DROP database itself. If TRUNCATE skip dropping/erasing the database. - if (!truncate) - DatabaseCatalog::instance().detachDatabase(getContext(), database_name, drop, database->shouldBeEmptyOnDetach()); + for (const auto & table : tables_to_drop) + { + query_for_table.setTable(table.first.getTableName()); + query_for_table.is_dictionary = table.second; + DatabasePtr db; + UUID table_to_wait = UUIDHelpers::Nil; + executeToTableImpl(table_context, query_for_table, db, table_to_wait); + uuids_to_wait.push_back(table_to_wait); } } + // only if operation is DETACH + if ((!drop || !truncate) && query.sync) + { + /// Avoid "some tables are still in use" when sync mode is enabled + for (const auto & table_uuid : uuids_to_wait) + database->waitDetachedTableNotInUse(table_uuid); + } + + /// Protects from concurrent CREATE TABLE queries + auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name); + // only if operation is DETACH + if (!drop || !truncate) + database->assertCanBeDetached(true); + + /// DETACH or DROP database itself. If TRUNCATE skip dropping/erasing the database. + if (!truncate) + DatabaseCatalog::instance().detachDatabase(getContext(), database_name, drop, database->shouldBeEmptyOnDetach()); return {}; } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ca0e84a5267..b7359b85079 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -86,6 +86,7 @@ #include #include #include +#include #include #include #include @@ -1676,7 +1677,11 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

(query_plan.getCurrentDataStream(), expressions.array_join); + = std::make_unique( + query_plan.getCurrentDataStream(), + *expressions.array_join, + settings.enable_unaligned_array_join, + settings.max_block_size); array_join_step->setStepDescription("ARRAY JOIN"); query_plan.addStep(std::move(array_join_step)); diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 5b378087707..a82e8d76c4e 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -304,21 +304,30 @@ Session::~Session() LOG_DEBUG(log, "{} Logout, user_id: {}", toString(auth_id), toString(*user_id)); if (auto session_log = getSessionLog()) { - session_log->addLogOut(auth_id, user, getClientInfo()); + session_log->addLogOut(auth_id, user, user_authenticated_with, getClientInfo()); } } } -AuthenticationType Session::getAuthenticationType(const String & user_name) const +std::unordered_set Session::getAuthenticationTypes(const String & user_name) const { - return global_context->getAccessControl().read(user_name)->auth_data.getType(); + std::unordered_set authentication_types; + + const auto user_to_query = global_context->getAccessControl().read(user_name); + + for (const auto & authentication_method : user_to_query->authentication_methods) + { + authentication_types.insert(authentication_method.getType()); + } + + return authentication_types; } -AuthenticationType Session::getAuthenticationTypeOrLogInFailure(const String & user_name) const +std::unordered_set Session::getAuthenticationTypesOrLogInFailure(const String & user_name) const { try { - return getAuthenticationType(user_name); + return getAuthenticationTypes(user_name); } catch (const Exception & e) { @@ -354,6 +363,7 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So { auto auth_result = global_context->getAccessControl().authenticate(credentials_, address.host(), getClientInfo().getLastForwardedFor()); user_id = auth_result.user_id; + user_authenticated_with = auth_result.authentication_data; settings_from_auth_server = auth_result.settings; LOG_DEBUG(log, "{} Authenticated with global context as user {}", toString(auth_id), toString(*user_id)); @@ -698,7 +708,8 @@ void Session::recordLoginSuccess(ContextPtr login_context) const settings, access->getAccess(), getClientInfo(), - user); + user, + user_authenticated_with); } notified_session_log_about_login = true; diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index fc41c78e666..ab4bc53b6f1 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -43,10 +43,10 @@ public: Session & operator=(const Session &) = delete; /// Provides information about the authentication type of a specified user. - AuthenticationType getAuthenticationType(const String & user_name) const; + std::unordered_set getAuthenticationTypes(const String & user_name) const; /// Same as getAuthenticationType, but adds LoginFailure event in case of error. - AuthenticationType getAuthenticationTypeOrLogInFailure(const String & user_name) const; + std::unordered_set getAuthenticationTypesOrLogInFailure(const String & user_name) const; /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. /// The function throws an exception if there is no such user or password is wrong. @@ -113,6 +113,7 @@ private: mutable UserPtr user; std::optional user_id; + AuthenticationData user_authenticated_with; ContextMutablePtr session_context; mutable bool query_context_created = false; diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index 866f5ba8c0a..ef102f94308 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -214,7 +214,8 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, const Settings & settings, const ContextAccessPtr & access, const ClientInfo & client_info, - const UserPtr & login_user) + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with) { SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS); log_entry.client_info = client_info; @@ -222,9 +223,11 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, if (login_user) { log_entry.user = login_user->getName(); - log_entry.user_identified_with = login_user->auth_data.getType(); + log_entry.user_identified_with = user_authenticated_with.getType(); } - log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : ""; + + log_entry.external_auth_server = user_authenticated_with.getLDAPServerName(); + log_entry.session_id = session_id; @@ -256,15 +259,19 @@ void SessionLog::addLoginFailure( add(std::move(log_entry)); } -void SessionLog::addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info) +void SessionLog::addLogOut( + const UUID & auth_id, + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with, + const ClientInfo & client_info) { auto log_entry = SessionLogElement(auth_id, SESSION_LOGOUT); if (login_user) { log_entry.user = login_user->getName(); - log_entry.user_identified_with = login_user->auth_data.getType(); + log_entry.user_identified_with = user_authenticated_with.getType(); } - log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : ""; + log_entry.external_auth_server = user_authenticated_with.getLDAPServerName(); log_entry.client_info = client_info; add(std::move(log_entry)); diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h index 5bacb9677c0..6221267d14c 100644 --- a/src/Interpreters/SessionLog.h +++ b/src/Interpreters/SessionLog.h @@ -22,6 +22,7 @@ class ContextAccess; struct User; using UserPtr = std::shared_ptr; using ContextAccessPtr = std::shared_ptr; +class AuthenticationData; /** A struct which will be inserted as row into session_log table. * @@ -71,17 +72,21 @@ struct SessionLogElement class SessionLog : public SystemLog { using SystemLog::SystemLog; - public: void addLoginSuccess(const UUID & auth_id, const String & session_id, const Settings & settings, const ContextAccessPtr & access, const ClientInfo & client_info, - const UserPtr & login_user); + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with); void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional & user, const Exception & reason); - void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info); + void addLogOut( + const UUID & auth_id, + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with, + const ClientInfo & client_info); }; } diff --git a/src/Interpreters/ZooKeeperLog.cpp b/src/Interpreters/ZooKeeperLog.cpp index 0d3063a569e..769757a5fba 100644 --- a/src/Interpreters/ZooKeeperLog.cpp +++ b/src/Interpreters/ZooKeeperLog.cpp @@ -93,6 +93,7 @@ ColumnsDescription ZooKeeperLogElement::getColumnsDescription() {"FilteredList", static_cast(Coordination::OpNum::FilteredList)}, {"CheckNotExists", static_cast(Coordination::OpNum::CheckNotExists)}, {"CreateIfNotExists", static_cast(Coordination::OpNum::CreateIfNotExists)}, + {"RemoveRecursive", static_cast(Coordination::OpNum::RemoveRecursive)}, }); auto error_enum = getCoordinationErrorCodesEnumType(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 6c22e71bccf..3260ea890c6 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -99,6 +99,7 @@ namespace DB namespace ErrorCodes { extern const int QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS; + extern const int QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE; extern const int QUERY_CACHE_USED_WITH_SYSTEM_TABLE; extern const int INTO_OUTFILE_NOT_ALLOWED; extern const int INVALID_TRANSACTION; @@ -1121,6 +1122,21 @@ static std::tuple executeQueryImpl( && (ast->as() || ast->as()); QueryCache::Usage query_cache_usage = QueryCache::Usage::None; + /// Bug 67476: If the query runs with a non-THROW overflow mode and hits a limit, the query cache will store a truncated result (if + /// enabled). This is incorrect. Unfortunately it is hard to detect from the perspective of the query cache that the query result + /// is truncated. Therefore throw an exception, to notify the user to disable either the query cache or use another overflow mode. + if (settings.use_query_cache && (settings.read_overflow_mode != OverflowMode::THROW + || settings.read_overflow_mode_leaf != OverflowMode::THROW + || settings.group_by_overflow_mode != OverflowMode::THROW + || settings.sort_overflow_mode != OverflowMode::THROW + || settings.result_overflow_mode != OverflowMode::THROW + || settings.timeout_overflow_mode != OverflowMode::THROW + || settings.set_overflow_mode != OverflowMode::THROW + || settings.join_overflow_mode != OverflowMode::THROW + || settings.transfer_overflow_mode != OverflowMode::THROW + || settings.distinct_overflow_mode != OverflowMode::THROW)) + throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE, "use_query_cache and overflow_mode != 'throw' cannot be used together"); + /// If the query runs with "use_query_cache = 1", we first probe if the query cache already contains the query result (if yes: /// return result from cache). If doesn't, we execute the query normally and write the result into the query cache. Both steps use a /// hash of the AST, the current database and the settings as cache key. Unfortunately, the settings are in some places internally diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 386ed900960..75082041161 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -44,7 +44,7 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt { if (type && *type == AuthenticationType::NO_PASSWORD) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED" + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " no_password" << (settings.hilite ? IAST::hilite_none : ""); return; } @@ -160,12 +160,9 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = AuthenticationTypeInfo::get(*type).name; } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); - if (!auth_type_name.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH " << auth_type_name - << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << auth_type_name << (settings.hilite ? IAST::hilite_none : ""); } if (!prefix.empty()) diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 6f0ccc76797..ec48c32b684 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -19,9 +19,25 @@ namespace << quoteString(new_name); } - void formatAuthenticationData(const ASTAuthenticationData & auth_data, const IAST::FormatSettings & settings) + void formatAuthenticationData(const std::vector> & authentication_methods, const IAST::FormatSettings & settings) { - auth_data.format(settings); + // safe because this method is only called if authentication_methods.size > 1 + // if the first type is present, include the `WITH` keyword + if (authentication_methods[0]->type) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH" << (settings.hilite ? IAST::hilite_none : ""); + } + + for (std::size_t i = 0; i < authentication_methods.size(); i++) + { + authentication_methods[i]->format(settings); + + bool is_last = i < authentication_methods.size() - 1; + if (is_last) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : ","); + } + } } void formatValidUntil(const IAST & valid_until, const IAST::FormatSettings & settings) @@ -165,6 +181,7 @@ ASTPtr ASTCreateUserQuery::clone() const { auto res = std::make_shared(*this); res->children.clear(); + res->authentication_methods.clear(); if (names) res->names = std::static_pointer_cast(names->clone()); @@ -181,10 +198,11 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); - if (auth_data) + for (const auto & authentication_method : authentication_methods) { - res->auth_data = std::static_pointer_cast(auth_data->clone()); - res->children.push_back(res->auth_data); + auto ast_clone = std::static_pointer_cast(authentication_method->clone()); + res->authentication_methods.push_back(ast_clone); + res->children.push_back(ast_clone); } return res; @@ -223,8 +241,24 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (new_name) formatRenameTo(*new_name, format); - if (auth_data) - formatAuthenticationData(*auth_data, format); + if (authentication_methods.empty()) + { + // If identification (auth method) is missing from query, we should serialize it in the form of `NO_PASSWORD` unless it is alter query + if (!alter) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH no_password" << (format.hilite ? IAST::hilite_none : ""); + } + } + else + { + if (add_identified_with) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " ADD" << (format.hilite ? IAST::hilite_none : ""); + } + + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (format.hilite ? IAST::hilite_none : ""); + formatAuthenticationData(authentication_methods, format); + } if (valid_until) formatValidUntil(*valid_until, format); @@ -247,6 +281,9 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (grantees) formatGrantees(*grantees, format); + + if (reset_authentication_methods_to_new) + format.ostr << (format.hilite ? hilite_keyword : "") << " RESET AUTHENTICATION METHODS TO NEW" << (format.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 4e14d86c425..e1bae98f2f3 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -42,12 +42,15 @@ public: bool if_exists = false; bool if_not_exists = false; bool or_replace = false; + bool reset_authentication_methods_to_new = false; + bool add_identified_with = false; + bool replace_authentication_methods = false; std::shared_ptr names; std::optional new_name; String storage_name; - std::shared_ptr auth_data; + std::vector> authentication_methods; std::optional hosts; std::optional add_hosts; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index d4a8813e9e4..8bfc84a28a6 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -43,21 +43,16 @@ namespace }); } - bool parseAuthenticationData(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data) + bool parseAuthenticationData( + IParserBase::Pos & pos, + Expected & expected, + std::shared_ptr & auth_data, + bool is_type_specifier_mandatory, + bool is_type_specifier_allowed, + bool should_parse_no_password) { return IParserBase::wrapParseImpl(pos, [&] { - if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) - { - auth_data = std::make_shared(); - auth_data->type = AuthenticationType::NO_PASSWORD; - - return true; - } - - if (!ParserKeyword{Keyword::IDENTIFIED}.ignore(pos, expected)) - return false; - std::optional type; bool expect_password = false; @@ -68,51 +63,65 @@ namespace bool expect_public_ssh_key = false; bool expect_http_auth_server = false; - if (ParserKeyword{Keyword::WITH}.ignore(pos, expected)) + auto parse_non_password_based_type = [&](auto check_type) { - for (auto check_type : collections::range(AuthenticationType::MAX)) + if (ParserKeyword{AuthenticationTypeInfo::get(check_type).keyword}.ignore(pos, expected)) { - if (ParserKeyword{AuthenticationTypeInfo::get(check_type).keyword}.ignore(pos, expected)) - { - type = check_type; + type = check_type; - if (check_type == AuthenticationType::LDAP) - expect_ldap_server_name = true; - else if (check_type == AuthenticationType::KERBEROS) - expect_kerberos_realm = true; - else if (check_type == AuthenticationType::SSL_CERTIFICATE) - expect_ssl_cert_subjects = true; - else if (check_type == AuthenticationType::SSH_KEY) - expect_public_ssh_key = true; - else if (check_type == AuthenticationType::HTTP) - expect_http_auth_server = true; - else if (check_type != AuthenticationType::NO_PASSWORD) - expect_password = true; + if (check_type == AuthenticationType::LDAP) + expect_ldap_server_name = true; + else if (check_type == AuthenticationType::KERBEROS) + expect_kerberos_realm = true; + else if (check_type == AuthenticationType::SSL_CERTIFICATE) + expect_ssl_cert_subjects = true; + else if (check_type == AuthenticationType::SSH_KEY) + expect_public_ssh_key = true; + else if (check_type == AuthenticationType::HTTP) + expect_http_auth_server = true; + else if (check_type != AuthenticationType::NO_PASSWORD) + expect_password = true; + return true; + } + + return false; + }; + + { + const auto first_authentication_type_element_to_check + = should_parse_no_password ? AuthenticationType::NO_PASSWORD : AuthenticationType::PLAINTEXT_PASSWORD; + + for (auto check_type : collections::range(first_authentication_type_element_to_check, AuthenticationType::MAX)) + { + if (parse_non_password_based_type(check_type)) break; - } } + } - if (!type) + if (!type) + { + if (ParserKeyword{Keyword::SHA256_HASH}.ignore(pos, expected)) { - if (ParserKeyword{Keyword::SHA256_HASH}.ignore(pos, expected)) - { - type = AuthenticationType::SHA256_PASSWORD; - expect_hash = true; - } - else if (ParserKeyword{Keyword::DOUBLE_SHA1_HASH}.ignore(pos, expected)) - { - type = AuthenticationType::DOUBLE_SHA1_PASSWORD; - expect_hash = true; - } - else if (ParserKeyword{Keyword::BCRYPT_HASH}.ignore(pos, expected)) - { - type = AuthenticationType::BCRYPT_PASSWORD; - expect_hash = true; - } - else - return false; + type = AuthenticationType::SHA256_PASSWORD; + expect_hash = true; } + else if (ParserKeyword{Keyword::DOUBLE_SHA1_HASH}.ignore(pos, expected)) + { + type = AuthenticationType::DOUBLE_SHA1_PASSWORD; + expect_hash = true; + } + else if (ParserKeyword{Keyword::BCRYPT_HASH}.ignore(pos, expected)) + { + type = AuthenticationType::BCRYPT_PASSWORD; + expect_hash = true; + } + else if (is_type_specifier_mandatory) + return false; + } + else if (!is_type_specifier_allowed) + { + return false; } /// If authentication type is not specified, then the default password type is used @@ -219,6 +228,69 @@ namespace } + bool parseIdentifiedWith( + IParserBase::Pos & pos, + Expected & expected, + std::vector> & authentication_methods, + bool should_parse_no_password) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{Keyword::IDENTIFIED}.ignore(pos, expected)) + return false; + + // Parse first authentication method which doesn't come with a leading comma + { + bool is_type_specifier_mandatory = ParserKeyword{Keyword::WITH}.ignore(pos, expected); + + std::shared_ptr ast_authentication_data; + + if (!parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory, is_type_specifier_mandatory, should_parse_no_password)) + { + return false; + } + + authentication_methods.push_back(ast_authentication_data); + } + + // Need to save current position, process comma and only update real position in case there is an authentication method after + // the comma. Otherwise, position should not be changed as it needs to be processed by other parsers and possibly throw error + // on trailing comma. + IParserBase::Pos aux_pos = pos; + while (ParserToken{TokenType::Comma}.ignore(aux_pos, expected)) + { + std::shared_ptr ast_authentication_data; + + if (!parseAuthenticationData(aux_pos, expected, ast_authentication_data, false, true, should_parse_no_password)) + { + break; + } + + pos = aux_pos; + authentication_methods.push_back(ast_authentication_data); + } + + return !authentication_methods.empty(); + }); + } + + bool parseIdentifiedOrNotIdentified(IParserBase::Pos & pos, Expected & expected, std::vector> & authentication_methods) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) + { + authentication_methods.emplace_back(std::make_shared()); + authentication_methods.back()->type = AuthenticationType::NO_PASSWORD; + + return true; + } + + return parseIdentifiedWith(pos, expected, authentication_methods, true); + }); + } + + bool parseHostsWithoutPrefix(IParserBase::Pos & pos, Expected & expected, AllowedClientHosts & hosts) { AllowedClientHosts res_hosts; @@ -411,6 +483,27 @@ namespace return until_p.parse(pos, valid_until, expected); }); } + + bool parseAddIdentifiedWith(IParserBase::Pos & pos, Expected & expected, std::vector> & auth_data) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{Keyword::ADD}.ignore(pos, expected)) + { + return false; + } + + return parseIdentifiedWith(pos, expected, auth_data, false); + }); + } + + bool parseResetAuthenticationMethods(IParserBase::Pos & pos, Expected & expected) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{Keyword::RESET_AUTHENTICATION_METHODS_TO_NEW}.ignore(pos, expected); + }); + } } @@ -456,7 +549,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec std::optional hosts; std::optional add_hosts; std::optional remove_hosts; - std::shared_ptr auth_data; + std::vector> auth_data; std::shared_ptr default_roles; std::shared_ptr settings; std::shared_ptr grantees; @@ -464,19 +557,28 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ASTPtr valid_until; String cluster; String storage_name; + bool reset_authentication_methods_to_new = false; + + bool parsed_identified_with = false; + bool parsed_add_identified_with = false; while (true) { - if (!auth_data) + if (auth_data.empty() && !reset_authentication_methods_to_new) { - std::shared_ptr new_auth_data; - if (parseAuthenticationData(pos, expected, new_auth_data)) + parsed_identified_with = parseIdentifiedOrNotIdentified(pos, expected, auth_data); + + if (!parsed_identified_with && alter) { - auth_data = std::move(new_auth_data); - continue; + parsed_add_identified_with = parseAddIdentifiedWith(pos, expected, auth_data); } } + if (!reset_authentication_methods_to_new && alter && auth_data.empty()) + { + reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); + } + if (!valid_until) { parseValidUntil(pos, expected, valid_until); @@ -564,7 +666,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->cluster = std::move(cluster); query->names = std::move(names); query->new_name = std::move(new_name); - query->auth_data = std::move(auth_data); + query->authentication_methods = std::move(auth_data); query->hosts = std::move(hosts); query->add_hosts = std::move(add_hosts); query->remove_hosts = std::move(remove_hosts); @@ -574,9 +676,14 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->default_database = std::move(default_database); query->valid_until = std::move(valid_until); query->storage_name = std::move(storage_name); + query->reset_authentication_methods_to_new = reset_authentication_methods_to_new; + query->add_identified_with = parsed_add_identified_with; + query->replace_authentication_methods = parsed_identified_with; - if (query->auth_data) - query->children.push_back(query->auth_data); + for (const auto & authentication_method : query->authentication_methods) + { + query->children.push_back(authentication_method); + } if (query->valid_until) query->children.push_back(query->valid_until); diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index ab0e70eb0e5..46e08cf3f7e 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -407,6 +407,7 @@ namespace DB MR_MACROS(REPLACE_PARTITION, "REPLACE PARTITION") \ MR_MACROS(REPLACE, "REPLACE") \ MR_MACROS(RESET_SETTING, "RESET SETTING") \ + MR_MACROS(RESET_AUTHENTICATION_METHODS_TO_NEW, "RESET AUTHENTICATION METHODS TO NEW") \ MR_MACROS(RESPECT_NULLS, "RESPECT NULLS") \ MR_MACROS(RESTORE, "RESTORE") \ MR_MACROS(RESTRICT, "RESTRICT") \ diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 002ad94f6ea..7836a863920 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -1,10 +1,42 @@ #pragma once -#include +#include +#include +#include +#include + namespace DB { +class AbstractFunction +{ + friend class FunctionSecretArgumentsFinder; +public: + class Argument + { + public: + virtual ~Argument() = default; + virtual std::unique_ptr getFunction() const = 0; + virtual bool isIdentifier() const = 0; + virtual bool tryGetString(String * res, bool allow_identifier) const = 0; + }; + class Arguments + { + public: + virtual ~Arguments() = default; + virtual size_t size() const = 0; + virtual std::unique_ptr at(size_t n) const = 0; + }; + + virtual ~AbstractFunction() = default; + virtual String name() const = 0; + bool hasArguments() const { return !!arguments; } + +protected: + std::unique_ptr arguments; +}; + class FunctionSecretArgumentsFinder { public: @@ -23,6 +55,485 @@ public: return count != 0 || !nested_maps.empty(); } }; + + explicit FunctionSecretArgumentsFinder(std::unique_ptr && function_) : function(std::move(function_)) {} + + FunctionSecretArgumentsFinder::Result getResult() const { return result; } + +protected: + const std::unique_ptr function; + Result result; + + void markSecretArgument(size_t index, bool argument_is_named = false) + { + if (index >= function->arguments->size()) + return; + if (!result.count) + { + result.start = index; + result.are_named = argument_is_named; + } + chassert(index >= result.start); /// We always check arguments consecutively + result.count = index + 1 - result.start; + if (!argument_is_named) + result.are_named = false; + } + + void findOrdinaryFunctionSecretArguments() + { + if ((function->name() == "mysql") || (function->name() == "postgresql") || (function->name() == "mongodb")) + { + /// mysql('host:port', 'database', 'table', 'user', 'password', ...) + /// postgresql('host:port', 'database', 'table', 'user', 'password', ...) + /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...) + findMySQLFunctionSecretArguments(); + } + else if ((function->name() == "s3") || (function->name() == "cosn") || (function->name() == "oss") || + (function->name() == "deltaLake") || (function->name() == "hudi") || (function->name() == "iceberg") || + (function->name() == "gcs")) + { + /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) + findS3FunctionSecretArguments(/* is_cluster_function= */ false); + } + else if (function->name() == "s3Cluster") + { + /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...) + findS3FunctionSecretArguments(/* is_cluster_function= */ true); + } + else if (function->name() == "azureBlobStorage") + { + /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) + findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ false); + } + else if (function->name() == "azureBlobStorageCluster") + { + /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) + findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ true); + } + else if ((function->name() == "remote") || (function->name() == "remoteSecure")) + { + /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...) + findRemoteFunctionSecretArguments(); + } + else if ((function->name() == "encrypt") || (function->name() == "decrypt") || + (function->name() == "aes_encrypt_mysql") || (function->name() == "aes_decrypt_mysql") || + (function->name() == "tryDecrypt")) + { + /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) + findEncryptionFunctionSecretArguments(); + } + else if (function->name() == "url") + { + findURLSecretArguments(); + } + } + + void findMySQLFunctionSecretArguments() + { + if (isNamedCollectionName(0)) + { + /// mysql(named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 1); + } + else + { + /// mysql('host:port', 'database', 'table', 'user', 'password', ...) + markSecretArgument(4); + } + } + + /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should + /// always be at the end). Marks "headers" as secret, if found. + size_t excludeS3OrURLNestedMaps() + { + size_t count = function->arguments->size(); + while (count > 0) + { + const auto f = function->arguments->at(count - 1)->getFunction(); + if (!f) + break; + if (f->name() == "headers") + result.nested_maps.push_back(f->name()); + else if (f->name() != "extra_credentials") + break; + count -= 1; + } + return count; + } + + void findS3FunctionSecretArguments(bool is_cluster_function) + { + /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. + size_t url_arg_idx = is_cluster_function ? 1 : 0; + + if (!is_cluster_function && isNamedCollectionName(0)) + { + /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...) + findSecretNamedArgument("secret_access_key", 1); + return; + } + + /// We should check other arguments first because we don't need to do any replacement in case of + /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) + /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) + size_t count = excludeS3OrURLNestedMaps(); + if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4)) + { + String second_arg; + if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg)) + { + if (boost::iequals(second_arg, "NOSIGN")) + return; /// The argument after 'url' is "NOSIGN". + + if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) + return; /// The argument after 'url' is a format: s3('url', 'format', ...) + } + } + + /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: + /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) + /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') + if (url_arg_idx + 2 < count) + markSecretArgument(url_arg_idx + 2); + } + + void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function) + { + /// azureBlobStorage('cluster_name', 'conn_string/storage_account_url', ...) has 'conn_string/storage_account_url' as its second argument. + size_t url_arg_idx = is_cluster_function ? 1 : 0; + + if (!is_cluster_function && isNamedCollectionName(0)) + { + /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + findSecretNamedArgument("account_key", 1); + return; + } + else if (is_cluster_function && isNamedCollectionName(1)) + { + /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) + findSecretNamedArgument("account_key", 2); + return; + } + + /// We should check other arguments first because we don't need to do any replacement in case storage_account_url is not used + /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) + /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) + size_t count = function->arguments->size(); + if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) + { + String second_arg; + if (tryGetStringFromArgument(url_arg_idx + 3, &second_arg)) + { + if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) + return; /// The argument after 'url' is a format: s3('url', 'format', ...) + } + } + + /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature + if (url_arg_idx + 4 < count) + markSecretArgument(url_arg_idx + 4); + } + + void findURLSecretArguments() + { + if (!isNamedCollectionName(0)) + excludeS3OrURLNestedMaps(); + } + + bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const + { + if (arg_idx >= function->arguments->size()) + return false; + + return tryGetStringFromArgument(*function->arguments->at(arg_idx), res, allow_identifier); + } + + static bool tryGetStringFromArgument(const AbstractFunction::Argument & argument, String * res, bool allow_identifier = true) + { + return argument.tryGetString(res, allow_identifier); + } + + void findRemoteFunctionSecretArguments() + { + if (isNamedCollectionName(0)) + { + /// remote(named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 1); + return; + } + + /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: + /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key]) + /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key]) + /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key]) + + /// But we should check the number of arguments first because we don't need to do any replacements in case of + /// remote('addresses_expr', db.table) + if (function->arguments->size() < 3) + return; + + size_t arg_num = 1; + + /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'. + auto table_function = function->arguments->at(arg_num)->getFunction(); + if (table_function && KnownTableFunctionNames::instance().exists(table_function->name())) + { + ++arg_num; + } + else + { + std::optional database; + std::optional qualified_table_name; + if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name)) + { + /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'. + /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user' + /// before the argument 'password'. So it's safer to wipe two arguments just in case. + /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string + /// before wiping it (because the `password` argument is always a literal string). + if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false)) + { + /// Wipe either `password` or `user`. + markSecretArgument(arg_num + 2); + } + if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false)) + { + /// Wipe either `password` or `sharding_key`. + markSecretArgument(arg_num + 3); + } + return; + } + + /// Skip the current argument (which is either a database name or a qualified table name). + ++arg_num; + if (database) + { + /// Skip the 'table' argument if the previous argument was a database name. + ++arg_num; + } + } + + /// Skip username. + ++arg_num; + + /// Do our replacement: + /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...) + /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string + /// before wiping it (because the `password` argument is always a literal string). + bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false); + if (can_be_password) + markSecretArgument(arg_num); + } + + /// Tries to get either a database name or a qualified table name from an argument. + /// Empty string is also allowed (it means the default database). + /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password. + bool tryGetDatabaseNameOrQualifiedTableName( + size_t arg_idx, + std::optional & res_database, + std::optional & res_qualified_table_name) const + { + res_database.reset(); + res_qualified_table_name.reset(); + + String str; + if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true)) + return false; + + if (str.empty()) + { + res_database = ""; + return true; + } + + auto qualified_table_name = QualifiedTableName::tryParseFromString(str); + if (!qualified_table_name) + return false; + + if (qualified_table_name->database.empty()) + res_database = std::move(qualified_table_name->table); + else + res_qualified_table_name = std::move(qualified_table_name); + return true; + } + + void findEncryptionFunctionSecretArguments() + { + if (function->arguments->size() == 0) + return; + + /// We replace all arguments after 'mode' with '[HIDDEN]': + /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]') + result.start = 1; + result.count = function->arguments->size() - 1; + } + + void findTableEngineSecretArguments() + { + const String & engine_name = function->name(); + if (engine_name == "ExternalDistributed") + { + /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') + findExternalDistributedTableEngineSecretArguments(); + } + else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") || + (engine_name == "MaterializedPostgreSQL") || (engine_name == "MongoDB")) + { + /// MySQL('host:port', 'database', 'table', 'user', 'password', ...) + /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) + /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) + /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...) + findMySQLFunctionSecretArguments(); + } + else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") || + (engine_name == "DeltaLake") || (engine_name == "Hudi") || (engine_name == "Iceberg") || (engine_name == "S3Queue")) + { + /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...) + findS3TableEngineSecretArguments(); + } + else if (engine_name == "URL") + { + findURLSecretArguments(); + } + } + + void findExternalDistributedTableEngineSecretArguments() + { + if (isNamedCollectionName(1)) + { + /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 2); + } + else + { + /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') + markSecretArgument(5); + } + } + + void findS3TableEngineSecretArguments() + { + if (isNamedCollectionName(0)) + { + /// S3(named_collection, ..., secret_access_key = 'secret_access_key') + findSecretNamedArgument("secret_access_key", 1); + return; + } + + /// We should check other arguments first because we don't need to do any replacement in case of + /// S3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) + /// S3('url', 'format', 'compression' [, extra_credentials(..)] [, headers(..)]) + size_t count = excludeS3OrURLNestedMaps(); + if ((3 <= count) && (count <= 4)) + { + String second_arg; + if (tryGetStringFromArgument(1, &second_arg)) + { + if (boost::iequals(second_arg, "NOSIGN")) + return; /// The argument after 'url' is "NOSIGN". + + if (count == 3) + { + if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) + return; /// The argument after 'url' is a format: S3('url', 'format', ...) + } + } + } + + /// We replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: + /// S3('url', 'aws_access_key_id', 'aws_secret_access_key') + /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') + /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') + if (2 < count) + markSecretArgument(2); + } + + void findDatabaseEngineSecretArguments() + { + const String & engine_name = function->name(); + if ((engine_name == "MySQL") || (engine_name == "MaterializeMySQL") || + (engine_name == "MaterializedMySQL") || (engine_name == "PostgreSQL") || + (engine_name == "MaterializedPostgreSQL")) + { + /// MySQL('host:port', 'database', 'user', 'password') + /// PostgreSQL('host:port', 'database', 'user', 'password') + findMySQLDatabaseSecretArguments(); + } + else if (engine_name == "S3") + { + /// S3('url', 'access_key_id', 'secret_access_key') + findS3DatabaseSecretArguments(); + } + } + + void findMySQLDatabaseSecretArguments() + { + if (isNamedCollectionName(0)) + { + /// MySQL(named_collection, ..., password = 'password', ...) + findSecretNamedArgument("password", 1); + } + else + { + /// MySQL('host:port', 'database', 'user', 'password') + markSecretArgument(3); + } + } + + void findS3DatabaseSecretArguments() + { + if (isNamedCollectionName(0)) + { + /// S3(named_collection, ..., secret_access_key = 'password', ...) + findSecretNamedArgument("secret_access_key", 1); + } + else + { + /// S3('url', 'access_key_id', 'secret_access_key') + markSecretArgument(2); + } + } + + void findBackupNameSecretArguments() + { + const String & engine_name = function->name(); + if (engine_name == "S3") + { + /// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key]) + markSecretArgument(2); + } + } + + /// Whether a specified argument can be the name of a named collection? + bool isNamedCollectionName(size_t arg_idx) const + { + if (function->arguments->size() <= arg_idx) + return false; + + return function->arguments->at(arg_idx)->isIdentifier(); + } + + /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified. + void findSecretNamedArgument(const std::string_view & key, size_t start = 0) + { + for (size_t i = start; i < function->arguments->size(); ++i) + { + const auto & argument = function->arguments->at(i); + const auto equals_func = argument->getFunction(); + if (!equals_func || (equals_func->name() != "equals")) + continue; + + if (!equals_func->arguments || equals_func->arguments->size() != 2) + continue; + + String found_key; + if (!tryGetStringFromArgument(*equals_func->arguments->at(0), &found_key)) + continue; + + if (found_key == key) + markSecretArgument(i, /* argument_is_named= */ true); + } + } }; } diff --git a/src/Parsers/FunctionSecretArgumentsFinderAST.h b/src/Parsers/FunctionSecretArgumentsFinderAST.h index 94da30922cc..a260c0d58da 100644 --- a/src/Parsers/FunctionSecretArgumentsFinderAST.h +++ b/src/Parsers/FunctionSecretArgumentsFinderAST.h @@ -1,35 +1,97 @@ #pragma once #include -#include #include #include #include -#include - -#include namespace DB { - -/// Finds arguments of a specified function which should not be displayed for most users for security reasons. -/// That involves passwords and secret keys. -class FunctionSecretArgumentsFinderAST +class FunctionAST : public AbstractFunction { public: - explicit FunctionSecretArgumentsFinderAST(const ASTFunction & function_) : function(function_) + class ArgumentAST : public Argument { - if (!function.arguments) + public: + explicit ArgumentAST(const IAST * argument_) : argument(argument_) {} + std::unique_ptr getFunction() const override + { + if (const auto * f = argument->as()) + return std::make_unique(*f); + return nullptr; + } + bool isIdentifier() const override { return argument->as(); } + bool tryGetString(String * res, bool allow_identifier) const override + { + if (const auto * literal = argument->as()) + { + if (literal->value.getType() != Field::Types::String) + return false; + if (res) + *res = literal->value.safeGet(); + return true; + } + + if (allow_identifier) + { + if (const auto * id = argument->as()) + { + if (res) + *res = id->name(); + return true; + } + } + + return false; + } + private: + const IAST * argument = nullptr; + }; + + class ArgumentsAST : public Arguments + { + public: + explicit ArgumentsAST(const ASTs * arguments_) : arguments(arguments_) {} + size_t size() const override { return arguments ? arguments->size() : 0; } + std::unique_ptr at(size_t n) const override + { + return std::make_unique(arguments->at(n).get()); + } + private: + const ASTs * arguments = nullptr; + }; + + explicit FunctionAST(const ASTFunction & function_) : function(&function_) + { + if (!function->arguments) return; - const auto * expr_list = function.arguments->as(); + const auto * expr_list = function->arguments->as(); if (!expr_list) return; - arguments = &expr_list->children; - switch (function.kind) + arguments = std::make_unique(&expr_list->children); + } + + String name() const override { return function->name; } +private: + const ASTFunction * function = nullptr; +}; + +/// Finds arguments of a specified function which should not be displayed for most users for security reasons. +/// That involves passwords and secret keys. +class FunctionSecretArgumentsFinderAST : public FunctionSecretArgumentsFinder +{ +public: + explicit FunctionSecretArgumentsFinderAST(const ASTFunction & function_) + : FunctionSecretArgumentsFinder(std::make_unique(function_)) + { + if (!function->hasArguments()) + return; + + switch (function_.kind) { case ASTFunction::Kind::ORDINARY_FUNCTION: findOrdinaryFunctionSecretArguments(); break; case ASTFunction::Kind::WINDOW_FUNCTION: break; @@ -43,506 +105,7 @@ public: } FunctionSecretArgumentsFinder::Result getResult() const { return result; } - -private: - const ASTFunction & function; - const ASTs * arguments = nullptr; - FunctionSecretArgumentsFinder::Result result; - - void markSecretArgument(size_t index, bool argument_is_named = false) - { - if (index >= arguments->size()) - return; - if (!result.count) - { - result.start = index; - result.are_named = argument_is_named; - } - chassert(index >= result.start); /// We always check arguments consecutively - result.count = index + 1 - result.start; - if (!argument_is_named) - result.are_named = false; - } - - void findOrdinaryFunctionSecretArguments() - { - if ((function.name == "mysql") || (function.name == "postgresql") || (function.name == "mongodb")) - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - /// postgresql('host:port', 'database', 'table', 'user', 'password', ...) - /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); - } - else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss") || - (function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg")) - { - /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ false); - } - else if (function.name == "s3Cluster") - { - /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...) - findS3FunctionSecretArguments(/* is_cluster_function= */ true); - } - else if (function.name == "azureBlobStorage") - { - /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) - findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ false); - } - else if (function.name == "azureBlobStorageCluster") - { - /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) - findAzureBlobStorageFunctionSecretArguments(/* is_cluster_function= */ true); - } - else if ((function.name == "remote") || (function.name == "remoteSecure")) - { - /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...) - findRemoteFunctionSecretArguments(); - } - else if ((function.name == "encrypt") || (function.name == "decrypt") || - (function.name == "aes_encrypt_mysql") || (function.name == "aes_decrypt_mysql") || - (function.name == "tryDecrypt")) - { - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) - findEncryptionFunctionSecretArguments(); - } - else if (function.name == "url") - { - findURLSecretArguments(); - } - } - - void findMySQLFunctionSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// mysql(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - } - else - { - /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - markSecretArgument(4); - } - } - - /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should - /// always be at the end). Marks "headers" as secret, if found. - size_t excludeS3OrURLNestedMaps() - { - size_t count = arguments->size(); - while (count > 0) - { - const ASTFunction * f = arguments->at(count - 1)->as(); - if (!f) - break; - if (f->name == "headers") - result.nested_maps.push_back(f->name); - else if (f->name != "extra_credentials") - break; - count -= 1; - } - return count; - } - - void findS3FunctionSecretArguments(bool is_cluster_function) - { - /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument. - size_t url_arg_idx = is_cluster_function ? 1 : 0; - - if (!is_cluster_function && isNamedCollectionName(0)) - { - /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...) - findSecretNamedArgument("secret_access_key", 1); - return; - } - - /// We should check other arguments first because we don't need to do any replacement in case of - /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - size_t count = excludeS3OrURLNestedMaps(); - if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4)) - { - String second_arg; - if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg)) - { - if (boost::iequals(second_arg, "NOSIGN")) - return; /// The argument after 'url' is "NOSIGN". - - if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) - return; /// The argument after 'url' is a format: s3('url', 'format', ...) - } - } - - /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: - /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) - /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - if (url_arg_idx + 2 < count) - markSecretArgument(url_arg_idx + 2); - } - - void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function) - { - /// azureBlobStorage('cluster_name', 'conn_string/storage_account_url', ...) has 'conn_string/storage_account_url' as its second argument. - size_t url_arg_idx = is_cluster_function ? 1 : 0; - - if (!is_cluster_function && isNamedCollectionName(0)) - { - /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) - findSecretNamedArgument("account_key", 1); - return; - } - else if (is_cluster_function && isNamedCollectionName(1)) - { - /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) - findSecretNamedArgument("account_key", 2); - return; - } - - /// We should check other arguments first because we don't need to do any replacement in case storage_account_url is not used - /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) - /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]) - size_t count = arguments->size(); - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) - { - String second_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &second_arg)) - { - if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) - return; /// The argument after 'url' is a format: s3('url', 'format', ...) - } - } - - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); - } - - void findURLSecretArguments() - { - if (!isNamedCollectionName(0)) - excludeS3OrURLNestedMaps(); - } - - bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const - { - if (arg_idx >= arguments->size()) - return false; - - return tryGetStringFromArgument(*(*arguments)[arg_idx], res, allow_identifier); - } - - static bool tryGetStringFromArgument(const IAST & argument, String * res, bool allow_identifier = true) - { - if (const auto * literal = argument.as()) - { - if (literal->value.getType() != Field::Types::String) - return false; - if (res) - *res = literal->value.safeGet(); - return true; - } - - if (allow_identifier) - { - if (const auto * id = argument.as()) - { - if (res) - *res = id->name(); - return true; - } - } - - return false; - } - - void findRemoteFunctionSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// remote(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - return; - } - - /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: - /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key]) - /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key]) - /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key]) - - /// But we should check the number of arguments first because we don't need to do any replacements in case of - /// remote('addresses_expr', db.table) - if (arguments->size() < 3) - return; - - size_t arg_num = 1; - - /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'. - const auto * table_function = (*arguments)[arg_num]->as(); - if (table_function && KnownTableFunctionNames::instance().exists(table_function->name)) - { - ++arg_num; - } - else - { - std::optional database; - std::optional qualified_table_name; - if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name)) - { - /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'. - /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user' - /// before the argument 'password'. So it's safer to wipe two arguments just in case. - /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string - /// before wiping it (because the `password` argument is always a literal string). - if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false)) - { - /// Wipe either `password` or `user`. - markSecretArgument(arg_num + 2); - } - if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false)) - { - /// Wipe either `password` or `sharding_key`. - markSecretArgument(arg_num + 3); - } - return; - } - - /// Skip the current argument (which is either a database name or a qualified table name). - ++arg_num; - if (database) - { - /// Skip the 'table' argument if the previous argument was a database name. - ++arg_num; - } - } - - /// Skip username. - ++arg_num; - - /// Do our replacement: - /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...) - /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string - /// before wiping it (because the `password` argument is always a literal string). - bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false); - if (can_be_password) - markSecretArgument(arg_num); - } - - /// Tries to get either a database name or a qualified table name from an argument. - /// Empty string is also allowed (it means the default database). - /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password. - bool tryGetDatabaseNameOrQualifiedTableName( - size_t arg_idx, - std::optional & res_database, - std::optional & res_qualified_table_name) const - { - res_database.reset(); - res_qualified_table_name.reset(); - - String str; - if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true)) - return false; - - if (str.empty()) - { - res_database = ""; - return true; - } - - auto qualified_table_name = QualifiedTableName::tryParseFromString(str); - if (!qualified_table_name) - return false; - - if (qualified_table_name->database.empty()) - res_database = std::move(qualified_table_name->table); - else - res_qualified_table_name = std::move(qualified_table_name); - return true; - } - - void findEncryptionFunctionSecretArguments() - { - if (arguments->empty()) - return; - - /// We replace all arguments after 'mode' with '[HIDDEN]': - /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]') - result.start = 1; - result.count = arguments->size() - 1; - } - - void findTableEngineSecretArguments() - { - const String & engine_name = function.name; - if (engine_name == "ExternalDistributed") - { - /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') - findExternalDistributedTableEngineSecretArguments(); - } - else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") || - (engine_name == "MaterializedPostgreSQL") || (engine_name == "MongoDB")) - { - /// MySQL('host:port', 'database', 'table', 'user', 'password', ...) - /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) - /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...) - /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...) - findMySQLFunctionSecretArguments(); - } - else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") || - (engine_name == "DeltaLake") || (engine_name == "Hudi") || (engine_name == "Iceberg") || (engine_name == "S3Queue")) - { - /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...) - findS3TableEngineSecretArguments(); - } - else if (engine_name == "URL") - { - findURLSecretArguments(); - } - } - - void findExternalDistributedTableEngineSecretArguments() - { - if (isNamedCollectionName(1)) - { - /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 2); - } - else - { - /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password') - markSecretArgument(5); - } - } - - void findS3TableEngineSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// S3(named_collection, ..., secret_access_key = 'secret_access_key') - findSecretNamedArgument("secret_access_key", 1); - return; - } - - /// We should check other arguments first because we don't need to do any replacement in case of - /// S3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)]) - /// S3('url', 'format', 'compression' [, extra_credentials(..)] [, headers(..)]) - size_t count = excludeS3OrURLNestedMaps(); - if ((3 <= count) && (count <= 4)) - { - String second_arg; - if (tryGetStringFromArgument(1, &second_arg)) - { - if (boost::iequals(second_arg, "NOSIGN")) - return; /// The argument after 'url' is "NOSIGN". - - if (count == 3) - { - if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg)) - return; /// The argument after 'url' is a format: S3('url', 'format', ...) - } - } - } - - /// We replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures: - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key') - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - if (2 < count) - markSecretArgument(2); - } - - void findDatabaseEngineSecretArguments() - { - const String & engine_name = function.name; - if ((engine_name == "MySQL") || (engine_name == "MaterializeMySQL") || - (engine_name == "MaterializedMySQL") || (engine_name == "PostgreSQL") || - (engine_name == "MaterializedPostgreSQL")) - { - /// MySQL('host:port', 'database', 'user', 'password') - /// PostgreSQL('host:port', 'database', 'user', 'password') - findMySQLDatabaseSecretArguments(); - } - else if (engine_name == "S3") - { - /// S3('url', 'access_key_id', 'secret_access_key') - findS3DatabaseSecretArguments(); - } - } - - void findMySQLDatabaseSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// MySQL(named_collection, ..., password = 'password', ...) - findSecretNamedArgument("password", 1); - } - else - { - /// MySQL('host:port', 'database', 'user', 'password') - markSecretArgument(3); - } - } - - void findS3DatabaseSecretArguments() - { - if (isNamedCollectionName(0)) - { - /// S3(named_collection, ..., secret_access_key = 'password', ...) - findSecretNamedArgument("secret_access_key", 1); - } - else - { - /// S3('url', 'access_key_id', 'secret_access_key') - markSecretArgument(2); - } - } - - void findBackupNameSecretArguments() - { - const String & engine_name = function.name; - if (engine_name == "S3") - { - /// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key]) - markSecretArgument(2); - } - } - - /// Whether a specified argument can be the name of a named collection? - bool isNamedCollectionName(size_t arg_idx) const - { - if (arguments->size() <= arg_idx) - return false; - - const auto * identifier = (*arguments)[arg_idx]->as(); - return identifier != nullptr; - } - - /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified. - void findSecretNamedArgument(const std::string_view & key, size_t start = 0) - { - for (size_t i = start; i < arguments->size(); ++i) - { - const auto & argument = (*arguments)[i]; - const auto * equals_func = argument->as(); - if (!equals_func || (equals_func->name != "equals")) - continue; - - const auto * expr_list = equals_func->arguments->as(); - if (!expr_list) - continue; - - const auto & equal_args = expr_list->children; - if (equal_args.size() != 2) - continue; - - String found_key; - if (!tryGetStringFromArgument(*equal_args[0], &found_key)) - continue; - - if (found_key == key) - markSecretArgument(i, /* argument_is_named= */ true); - } - } }; + } diff --git a/src/Parsers/ParserShowColumnsQuery.cpp b/src/Parsers/ParserShowColumnsQuery.cpp index 5d26d7bf1d4..9c31786ad57 100644 --- a/src/Parsers/ParserShowColumnsQuery.cpp +++ b/src/Parsers/ParserShowColumnsQuery.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include @@ -18,7 +18,6 @@ bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ASTPtr from1; ASTPtr from2; - String from1_str; String from2_str; auto query = std::make_shared(); @@ -43,25 +42,18 @@ bool ParserShowColumnsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe else return false; - tryGetIdentifierNameInto(from1, from1_str); - - bool abbreviated_form = from1_str.contains("."); // FROM database.table - if (abbreviated_form) - { - std::vector split; - boost::split(split, from1_str, boost::is_any_of(".")); - query->database = split[0]; - query->table = split[1]; - } + const auto * table_id = from1->as(); + if (!table_id) + return false; + query->table = table_id->shortName(); + if (table_id->compound()) + query->database = table_id->name_parts[0]; else { if (ParserKeyword(Keyword::FROM).ignore(pos, expected) || ParserKeyword(Keyword::IN).ignore(pos, expected)) if (!ParserIdentifier().parse(pos, from2, expected)) return false; - tryGetIdentifierNameInto(from2, from2_str); - - query->table = from1_str; query->database = from2_str; } diff --git a/src/Parsers/ParserShowIndexesQuery.cpp b/src/Parsers/ParserShowIndexesQuery.cpp index 495dfc5101f..6469d74b016 100644 --- a/src/Parsers/ParserShowIndexesQuery.cpp +++ b/src/Parsers/ParserShowIndexesQuery.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include @@ -17,7 +17,6 @@ bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ASTPtr from1; ASTPtr from2; - String from1_str; String from2_str; auto query = std::make_shared(); @@ -39,25 +38,18 @@ bool ParserShowIndexesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe else return false; - tryGetIdentifierNameInto(from1, from1_str); - - bool abbreviated_form = from1_str.contains("."); // FROM database.table - if (abbreviated_form) - { - std::vector split; - boost::split(split, from1_str, boost::is_any_of(".")); - query->database = split[0]; - query->table = split[1]; - } + const auto * table_id = from1->as(); + if (!table_id) + return false; + query->table = table_id->shortName(); + if (table_id->compound()) + query->database = table_id->name_parts[0]; else { if (ParserKeyword(Keyword::FROM).ignore(pos, expected) || ParserKeyword(Keyword::IN).ignore(pos, expected)) if (!ParserIdentifier().parse(pos, from2, expected)) return false; - tryGetIdentifierNameInto(from2, from2_str); - - query->table = from1_str; query->database = from2_str; } diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 47f7a54389b..d850423c118 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -87,7 +87,7 @@ TEST_P(ParserTest, parseQuery) { if (input_text.starts_with("ATTACH")) { - auto salt = (dynamic_cast(ast.get())->auth_data)->getSalt().value_or(""); + auto salt = (dynamic_cast(ast.get())->authentication_methods.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } else @@ -283,6 +283,18 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, + { + "CREATE USER user1 IDENTIFIED WITH no_password", + "CREATE USER user1 IDENTIFIED WITH no_password" + }, + { + "CREATE USER user1", + "CREATE USER user1 IDENTIFIED WITH no_password" + }, + { + "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'", + "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'" + }, { "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'" @@ -291,6 +303,10 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, + { + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'", + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'" + }, { "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'" @@ -298,6 +314,10 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, { "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123' SALT 'EFFD7F6B03B3EA68B8F86C1E91614DD50E42EB31EF7160524916444D58B5E264'", "throws Syntax error" + }, + { + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' IDENTIFIED WITH plaintext_password BY 'def123'", + "throws Only one identified with is permitted" } }))); diff --git a/src/Parsers/tests/gtest_common.cpp b/src/Parsers/tests/gtest_common.cpp index 8ff9400d8a2..594436a1714 100644 --- a/src/Parsers/tests/gtest_common.cpp +++ b/src/Parsers/tests/gtest_common.cpp @@ -63,7 +63,7 @@ TEST_P(ParserKQLTest, parseKQLQuery) { if (input_text.starts_with("ATTACH")) { - auto salt = (dynamic_cast(ast.get())->auth_data)->getSalt().value_or(""); + auto salt = (dynamic_cast(ast.get())->authentication_methods.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } else diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 76583c82583..e0668a0eb8e 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1674,11 +1674,12 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_ PlannerActionsVisitor actions_visitor(planner_context); std::unordered_set array_join_expressions_output_nodes; - NameSet array_join_column_names; + Names array_join_column_names; + array_join_column_names.reserve(array_join_node.getJoinExpressions().getNodes().size()); for (auto & array_join_expression : array_join_node.getJoinExpressions().getNodes()) { const auto & array_join_column_identifier = planner_context->getColumnNodeIdentifierOrThrow(array_join_expression); - array_join_column_names.insert(array_join_column_identifier); + array_join_column_names.push_back(array_join_column_identifier); auto & array_join_expression_column = array_join_expression->as(); auto expression_dag_index_nodes = actions_visitor.visit(array_join_action_dag, array_join_expression_column.getExpressionOrThrow()); @@ -1727,8 +1728,13 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_ drop_unused_columns_before_array_join_transform_step->setStepDescription("DROP unused columns before ARRAY JOIN"); plan.addStep(std::move(drop_unused_columns_before_array_join_transform_step)); - auto array_join_action = std::make_shared(array_join_column_names, array_join_node.isLeft(), planner_context->getQueryContext()); - auto array_join_step = std::make_unique(plan.getCurrentDataStream(), std::move(array_join_action)); + const auto & settings = planner_context->getQueryContext()->getSettingsRef(); + auto array_join_step = std::make_unique( + plan.getCurrentDataStream(), + ArrayJoin{std::move(array_join_column_names), array_join_node.isLeft()}, + settings.enable_unaligned_array_join, + settings.max_block_size); + array_join_step->setStepDescription("ARRAY JOIN"); plan.addStep(std::move(array_join_step)); diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 4a7a23158ff..3356adb5315 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -78,7 +78,9 @@ void ORCOutputStream::write(const void* buf, size_t length) } ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) - : IOutputFormat(header_, out_), format_settings{format_settings_}, output_stream(out_) + : IOutputFormat(header_, out_) + , format_settings{format_settings_} + , output_stream(out_) { for (const auto & type : header_.getDataTypes()) data_types.push_back(recursiveRemoveLowCardinality(type)); @@ -565,6 +567,7 @@ void ORCBlockOutputFormat::prepareWriter() schema = orc::createStructType(); options.setCompression(getORCCompression(format_settings.orc.output_compression_method)); options.setRowIndexStride(format_settings.orc.output_row_index_stride); + options.setDictionaryKeySizeThreshold(format_settings.orc.output_dictionary_key_size_threshold); size_t columns_count = header.columns(); for (size_t i = 0; i != columns_count; ++i) schema->addStructField(header.safeGetByPosition(i).name, getORCType(recursiveRemoveLowCardinality(data_types[i]))); diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 23a0a756f0d..94cb6ae2ee5 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -24,27 +24,30 @@ static ITransformingStep::Traits getTraits() }; } -ArrayJoinStep::ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPtr array_join_) +ArrayJoinStep::ArrayJoinStep(const DataStream & input_stream_, ArrayJoin array_join_, bool is_unaligned_, size_t max_block_size_) : ITransformingStep( input_stream_, - ArrayJoinTransform::transformHeader(input_stream_.header, array_join_), + ArrayJoinTransform::transformHeader(input_stream_.header, array_join_.columns), getTraits()) , array_join(std::move(array_join_)) + , is_unaligned(is_unaligned_) + , max_block_size(max_block_size_) { } void ArrayJoinStep::updateOutputStream() { output_stream = createOutputStream( - input_streams.front(), ArrayJoinTransform::transformHeader(input_streams.front().header, array_join), getDataStreamTraits()); + input_streams.front(), ArrayJoinTransform::transformHeader(input_streams.front().header, array_join.columns), getDataStreamTraits()); } void ArrayJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { + auto array_join_actions = std::make_shared(array_join.columns, array_join.is_left, is_unaligned, max_block_size); pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) { bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; - return std::make_shared(header, array_join, on_totals); + return std::make_shared(header, array_join_actions, on_totals); }); } @@ -53,8 +56,8 @@ void ArrayJoinStep::describeActions(FormatSettings & settings) const String prefix(settings.offset, ' '); bool first = true; - settings.out << prefix << (array_join->is_left ? "LEFT " : "") << "ARRAY JOIN "; - for (const auto & column : array_join->columns) + settings.out << prefix << (array_join.is_left ? "LEFT " : "") << "ARRAY JOIN "; + for (const auto & column : array_join.columns) { if (!first) settings.out << ", "; @@ -68,10 +71,10 @@ void ArrayJoinStep::describeActions(FormatSettings & settings) const void ArrayJoinStep::describeActions(JSONBuilder::JSONMap & map) const { - map.add("Left", array_join->is_left); + map.add("Left", array_join.is_left); auto columns_array = std::make_unique(); - for (const auto & column : array_join->columns) + for (const auto & column : array_join.columns) columns_array->add(column); map.add("Columns", std::move(columns_array)); diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index 2d9b2ebd0c8..1a049d5805e 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace DB { @@ -10,7 +11,7 @@ using ArrayJoinActionPtr = std::shared_ptr; class ArrayJoinStep : public ITransformingStep { public: - explicit ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPtr array_join_); + ArrayJoinStep(const DataStream & input_stream_, ArrayJoin array_join_, bool is_unaligned_, size_t max_block_size_); String getName() const override { return "ArrayJoin"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; @@ -18,12 +19,15 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; - const ArrayJoinActionPtr & arrayJoin() const { return array_join; } + const Names & getColumns() const { return array_join.columns; } + bool isLeft() const { return array_join.is_left; } private: void updateOutputStream() override; - ArrayJoinActionPtr array_join; + ArrayJoin array_join; + bool is_unaligned = false; + size_t max_block_size = DEFAULT_BLOCK_SIZE; }; } diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h index 3b5e4e06953..d99f9a7d1f1 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h @@ -12,12 +12,15 @@ namespace DB struct Settings; class QueryStatus; using QueryStatusPtr = std::shared_ptr; +struct ITemporaryFileLookup; +using TemporaryFileLookupPtr = std::shared_ptr; struct BuildQueryPipelineSettings { ExpressionActionsSettings actions_settings; QueryStatusPtr process_list_element; ProgressCallback progress_callback = nullptr; + TemporaryFileLookupPtr temporary_file_lookup; const ExpressionActionsSettings & getActionsSettings() const { return actions_settings; } static BuildQueryPipelineSettings fromContext(ContextPtr from); diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index b71326ff75b..63ea8880cca 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -520,13 +520,14 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * array_join = typeid_cast(child.get())) { - const auto & array_join_actions = array_join->arrayJoin(); - const auto & keys = array_join_actions->columns; + const auto & keys = array_join->getColumns(); + std::unordered_set keys_set(keys.begin(), keys.end()); + const auto & array_join_header = array_join->getInputStreams().front().header; Names allowed_inputs; for (const auto & column : array_join_header) - if (!keys.contains(column.name)) + if (!keys_set.contains(column.name)) allowed_inputs.push_back(column.name); if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_inputs)) diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index 0d4f2330119..8866bb99cbe 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -24,11 +24,11 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node if (!(expression_step || filter_step) || !array_join_step) return 0; - const auto & array_join = array_join_step->arrayJoin(); + const auto & array_join_columns = array_join_step->getColumns(); const auto & expression = expression_step ? expression_step->getExpression() : filter_step->getExpression(); - auto split_actions = expression.splitActionsBeforeArrayJoin(array_join->columns); + auto split_actions = expression.splitActionsBeforeArrayJoin(array_join_columns); /// No actions can be moved before ARRAY JOIN. if (split_actions.first.trivial()) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index ac7fcdcf83f..903d40509f5 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -231,13 +231,15 @@ void buildSortingDAG(QueryPlan::Node & node, std::optional & dag, Fi { /// Should ignore limit because ARRAY JOIN can reduce the number of rows in case of empty array. /// But in case of LEFT ARRAY JOIN the result number of rows is always bigger. - if (!array_join->arrayJoin()->is_left) + if (!array_join->isLeft()) limit = 0; - const auto & array_joined_columns = array_join->arrayJoin()->columns; + const auto & array_joined_columns = array_join->getColumns(); if (dag) { + std::unordered_set keys_set(array_joined_columns.begin(), array_joined_columns.end()); + /// Remove array joined columns from outputs. /// Types are changed after ARRAY JOIN, and we can't use this columns anyway. ActionsDAG::NodeRawConstPtrs outputs; @@ -245,7 +247,7 @@ void buildSortingDAG(QueryPlan::Node & node, std::optional & dag, Fi for (const auto & output : dag->getOutputs()) { - if (!array_joined_columns.contains(output->result_name)) + if (!keys_set.contains(output->result_name)) outputs.push_back(output); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 218f0a61a48..60eadb80496 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1,6 +1,8 @@ #include +#include #include +#include #include #include #include @@ -8,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -24,10 +29,11 @@ #include #include #include -#include #include -#include +#include +#include #include +#include #include #include #include @@ -41,11 +47,6 @@ #include #include #include -#include -#include -#include -#include -#include #include #include @@ -381,6 +382,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit .all_callback = all_ranges_callback.value(), .callback = read_task_callback.value(), .number_of_current_replica = number_of_current_replica.value_or(client_info.number_of_current_replica), + .total_nodes_count = context->getClusterForParallelReplicas()->getShardsInfo().at(0).getAllNodeCount(), }; /// We have a special logic for local replica. It has to read less data, because in some cases it should @@ -563,6 +565,7 @@ Pipe ReadFromMergeTree::readInOrder( .all_callback = all_ranges_callback.value(), .callback = read_task_callback.value(), .number_of_current_replica = number_of_current_replica.value_or(client_info.number_of_current_replica), + .total_nodes_count = context->getClusterForParallelReplicas()->getShardsInfo().at(0).getAllNodeCount(), }; auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; @@ -2009,33 +2012,6 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons { auto result = getAnalysisResult(); - if (is_parallel_reading_from_replicas && context->canUseParallelReplicasOnInitiator() - && context->getSettingsRef().parallel_replicas_local_plan) - { - CoordinationMode mode = CoordinationMode::Default; - switch (result.read_type) - { - case ReadFromMergeTree::ReadType::Default: - mode = CoordinationMode::Default; - break; - case ReadFromMergeTree::ReadType::InOrder: - mode = CoordinationMode::WithOrder; - break; - case ReadFromMergeTree::ReadType::InReverseOrder: - mode = CoordinationMode::ReverseOrder; - break; - case ReadFromMergeTree::ReadType::ParallelReplicas: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Read type can't be ParallelReplicas on initiator"); - } - - chassert(number_of_current_replica.has_value()); - chassert(all_ranges_callback.has_value()); - - /// initialize working set from local replica - all_ranges_callback.value()( - InitialAllRangesAnnouncement(mode, result.parts_with_ranges.getDescriptions(), number_of_current_replica.value())); - } - if (enable_remove_parts_from_snapshot_optimization) { /// Do not keep data parts in snapshot. diff --git a/src/Processors/QueryPlan/TemporaryFiles.h b/src/Processors/QueryPlan/TemporaryFiles.h new file mode 100644 index 00000000000..943a2c2b0a4 --- /dev/null +++ b/src/Processors/QueryPlan/TemporaryFiles.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +class WriteBuffer; +class ReadBuffer; + +/// Interface for accessing temporary files by some logical name (or id). +/// While building query pipeline processors can lookup temporary files by some id and use them for writing and/or reading temporary data +/// without knowing what exactly is behind the name: local file, memory buffer, object in cloud storage, etc. +struct ITemporaryFileLookup : boost::noncopyable +{ + virtual ~ITemporaryFileLookup() = default; + + /// Give the caller a temporary write buffer, but don't give away the ownership. + virtual WriteBuffer & getTemporaryFileForWriting(const String & file_id) = 0; + + /// Give the caller a temporary read buffer, it exclusively belongs to the caller. + /// Other callers can get their own read buffer for the same temporary file. + virtual std::unique_ptr getTemporaryFileForReading(const String & file_id) = 0; +}; + +using TemporaryFileLookupPtr = std::shared_ptr; + +} diff --git a/src/Processors/Transforms/ArrayJoinTransform.cpp b/src/Processors/Transforms/ArrayJoinTransform.cpp index 1304434d74e..ec1d77e1a1b 100644 --- a/src/Processors/Transforms/ArrayJoinTransform.cpp +++ b/src/Processors/Transforms/ArrayJoinTransform.cpp @@ -10,20 +10,26 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -Block ArrayJoinTransform::transformHeader(Block header, const ArrayJoinActionPtr & array_join) +template +Block transformHeaderImpl(Block header, const Container & array_join_columns) { auto columns = header.getColumnsWithTypeAndName(); - array_join->prepare(columns); + ArrayJoinAction::prepare(array_join_columns, columns); Block res{std::move(columns)}; res.setColumns(res.mutateColumns()); return res; } +Block ArrayJoinTransform::transformHeader(Block header, const Names & array_join_columns) +{ + return transformHeaderImpl(std::move(header), array_join_columns); +} + ArrayJoinTransform::ArrayJoinTransform( const Block & header_, ArrayJoinActionPtr array_join_, bool /*on_totals_*/) - : IInflatingTransform(header_, transformHeader(header_, array_join_)) + : IInflatingTransform(header_, transformHeaderImpl(header_, array_join_->columns)) , array_join(std::move(array_join_)) { /// TODO diff --git a/src/Processors/Transforms/ArrayJoinTransform.h b/src/Processors/Transforms/ArrayJoinTransform.h index 4219135982d..9ade337e676 100644 --- a/src/Processors/Transforms/ArrayJoinTransform.h +++ b/src/Processors/Transforms/ArrayJoinTransform.h @@ -22,7 +22,7 @@ public: String getName() const override { return "ArrayJoinTransform"; } - static Block transformHeader(Block header, const ArrayJoinActionPtr & array_join); + static Block transformHeader(Block header, const Names & array_join_columns); protected: void consume(Chunk chunk) override; diff --git a/src/Processors/Transforms/ColumnGathererTransform.cpp b/src/Processors/Transforms/ColumnGathererTransform.cpp index 52fa42fdb51..f266d5c2e2f 100644 --- a/src/Processors/Transforms/ColumnGathererTransform.cpp +++ b/src/Processors/Transforms/ColumnGathererTransform.cpp @@ -183,13 +183,14 @@ void ColumnGathererStream::consume(Input & input, size_t source_num) ColumnGathererTransform::ColumnGathererTransform( const Block & header, size_t num_inputs, - ReadBuffer & row_sources_buf_, + std::unique_ptr row_sources_buf_, size_t block_preferred_size_rows_, size_t block_preferred_size_bytes_, bool is_result_sparse_) : IMergingTransform( num_inputs, header, header, /*have_all_inputs_=*/ true, /*limit_hint_=*/ 0, /*always_read_till_end_=*/ false, - num_inputs, row_sources_buf_, block_preferred_size_rows_, block_preferred_size_bytes_, is_result_sparse_) + num_inputs, *row_sources_buf_, block_preferred_size_rows_, block_preferred_size_bytes_, is_result_sparse_) + , row_sources_buf_holder(std::move(row_sources_buf_)) , log(getLogger("ColumnGathererStream")) { if (header.columns() != 1) diff --git a/src/Processors/Transforms/ColumnGathererTransform.h b/src/Processors/Transforms/ColumnGathererTransform.h index fbc9a6bfcc6..ce2671ce0bf 100644 --- a/src/Processors/Transforms/ColumnGathererTransform.h +++ b/src/Processors/Transforms/ColumnGathererTransform.h @@ -115,7 +115,7 @@ public: ColumnGathererTransform( const Block & header, size_t num_inputs, - ReadBuffer & row_sources_buf_, + std::unique_ptr row_sources_buf_, size_t block_preferred_size_rows_, size_t block_preferred_size_bytes_, bool is_result_sparse_); @@ -124,6 +124,8 @@ public: protected: void onFinish() override; + + std::unique_ptr row_sources_buf_holder; /// Keep ownership of row_sources_buf while it's in use by ColumnGathererStream. LoggerPtr log; }; diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index df7b6e7fbd7..5b981fc7a87 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -34,8 +34,12 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu auto current = pdata->data.get(); if (!current) return -1; + return setCertificateCallback(ssl, current.get(), log); +} - if (current->certs_chain.empty()) +int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log) +{ + if (current_data->certs_chain.empty()) return -1; if (auto err = SSL_clear_chain_certs(ssl); err != 1) @@ -43,12 +47,12 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu LOG_ERROR(log, "Clear certificates {}", Poco::Net::Utility::getLastError()); return -1; } - if (auto err = SSL_use_certificate(ssl, const_cast(current->certs_chain[0].certificate())); err != 1) + if (auto err = SSL_use_certificate(ssl, const_cast(current_data->certs_chain[0].certificate())); err != 1) { LOG_ERROR(log, "Use certificate {}", Poco::Net::Utility::getLastError()); return -1; } - for (auto cert = current->certs_chain.begin() + 1; cert != current->certs_chain.end(); cert++) + for (auto cert = current_data->certs_chain.begin() + 1; cert != current_data->certs_chain.end(); cert++) { if (auto err = SSL_add1_chain_cert(ssl, const_cast(cert->certificate())); err != 1) { @@ -56,7 +60,7 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu return -1; } } - if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current->key))); err != 1) + if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current_data->key))); err != 1) { LOG_ERROR(log, "Use private key {}", Poco::Net::Utility::getLastError()); return -1; diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 7472d2f6baa..28737988fdd 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -104,6 +104,9 @@ private: mutable std::mutex data_mutex; }; +/// A callback for OpenSSL +int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log); + } #endif diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 946eaf8aea4..47af568838a 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -17,9 +17,7 @@ void WriteBufferFromHTTPServerResponse::startSendHeaders() { headers_started_sending = true; - if (response.getChunkedTransferEncoding()) - setChunked(); - else if (response.getContentLength() == Poco::Net::HTTPMessage::UNKNOWN_CONTENT_LENGTH) + if (!response.getChunkedTransferEncoding() && response.getContentLength() == Poco::Net::HTTPMessage::UNKNOWN_CONTENT_LENGTH) { /// In case there is no Content-Length we cannot use keep-alive, /// since there is no way to know when the server send all the @@ -134,6 +132,8 @@ WriteBufferFromHTTPServerResponse::WriteBufferFromHTTPServerResponse( , response(response_) , is_http_method_head(is_http_method_head_) { + if (response.getChunkedTransferEncoding()) + setChunked(); } diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 3deb09bae88..5debc23f81a 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -376,11 +376,16 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl { try { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible - // (if password is specified using double SHA1). Otherwise, SHA256 plugin is used. - if (session->getAuthenticationTypeOrLogInFailure(user_name) == DB::AuthenticationType::SHA256_PASSWORD) + const auto user_authentication_types = session->getAuthenticationTypesOrLogInFailure(user_name); + + for (const auto user_authentication_type : user_authentication_types) { - authPluginSSL(); + // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible + // (if password is specified using double SHA1). Otherwise, SHA256 plugin is used. + if (user_authentication_type == DB::AuthenticationType::SHA256_PASSWORD) + { + authPluginSSL(); + } } std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index d8c0765bca4..f0633eb1c70 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -90,15 +90,15 @@ static inline void trySendExceptionToClient( void StaticRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & /*write_event*/) { + applyHTTPResponseHeaders(response, http_response_headers_override); + + if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) + response.setChunkedTransferEncoding(true); + auto out = responseWriteBuffer(request, response); try { - applyHTTPResponseHeaders(response, http_response_headers_override); - - if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) - response.setChunkedTransferEncoding(true); - /// Workaround. Poco does not detect 411 Length Required case. if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && !request.hasContentLength()) throw Exception(ErrorCodes::HTTP_LENGTH_REQUIRED, diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 2b9a7295198..50963887752 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1270,7 +1270,7 @@ void TCPHandler::sendReadTaskRequestAssumeLocked() void TCPHandler::sendMergeTreeAllRangesAnnouncementAssumeLocked(InitialAllRangesAnnouncement announcement) { writeVarUInt(Protocol::Server::MergeTreeAllRangesAnnouncement, *out); - announcement.serialize(*out); + announcement.serialize(*out, client_parallel_replicas_protocol_version); out->finishChunk(); out->next(); @@ -1280,7 +1280,7 @@ void TCPHandler::sendMergeTreeAllRangesAnnouncementAssumeLocked(InitialAllRanges void TCPHandler::sendMergeTreeReadTaskRequestAssumeLocked(ParallelReadRequest request) { writeVarUInt(Protocol::Server::MergeTreeReadTaskRequest, *out); - request.serialize(*out); + request.serialize(*out, client_parallel_replicas_protocol_version); out->finishChunk(); out->next(); @@ -1592,7 +1592,17 @@ void TCPHandler::receiveHello() /// Perform handshake for SSH authentication if (is_ssh_based_auth) { - if (session->getAuthenticationTypeOrLogInFailure(user) != AuthenticationType::SSH_KEY) + const auto authentication_types = session->getAuthenticationTypesOrLogInFailure(user); + + bool user_supports_ssh_authentication = std::find_if( + authentication_types.begin(), + authentication_types.end(), + [](auto authentication_type) + { + return authentication_type == AuthenticationType::SSH_KEY; + }) != authentication_types.end(); + + if (!user_supports_ssh_authentication) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Expected authentication with SSH key"); if (client_tcp_protocol_version < DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION) @@ -1652,6 +1662,9 @@ void TCPHandler::receiveAddendum() readStringBinary(proto_send_chunked_cl, *in); readStringBinary(proto_recv_chunked_cl, *in); } + + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + readVarUInt(client_parallel_replicas_protocol_version, *in); } @@ -1679,6 +1692,8 @@ void TCPHandler::sendHello() writeVarUInt(VERSION_MAJOR, *out); writeVarUInt(VERSION_MINOR, *out); writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + writeVarUInt(DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION, *out); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) writeStringBinary(DateLUT::instance().getTimeZone(), *out); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) @@ -2133,7 +2148,7 @@ bool TCPHandler::receiveUnexpectedData(bool throw_exception) std::shared_ptr maybe_compressed_in; if (last_block_in.compression == Protocol::Compression::Enable) - maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true); + maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true, /* external_data */ query_kind != ClientInfo::QueryKind::SECONDARY_QUERY); else maybe_compressed_in = in; @@ -2157,7 +2172,7 @@ void TCPHandler::initBlockInput() /// with another codec that the rest of the data. Example: data sent by Distributed tables. if (state.compression == Protocol::Compression::Enable) - state.maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true); + state.maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true, /* external_data */ query_kind != ClientInfo::QueryKind::SECONDARY_QUERY); else state.maybe_compressed_in = in; diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index dca40e98920..3b6e0059a30 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -188,6 +188,7 @@ private: UInt64 client_version_minor = 0; UInt64 client_version_patch = 0; UInt32 client_tcp_protocol_version = 0; + UInt32 client_parallel_replicas_protocol_version = 0; String proto_send_chunked_cl = "notchunked"; String proto_recv_chunked_cl = "notchunked"; String quota_key; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index ef76bc691ec..68778243371 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1142,6 +1142,16 @@ bool AlterCommands::hasFullTextIndex(const StorageInMemoryMetadata & metadata) return false; } +bool AlterCommands::hasLegacyInvertedIndex(const StorageInMemoryMetadata & metadata) +{ + for (const auto & index : metadata.secondary_indices) + { + if (index.type == INVERTED_INDEX_NAME) + return true; + } + return false; +} + bool AlterCommands::hasVectorSimilarityIndex(const StorageInMemoryMetadata & metadata) { for (const auto & index : metadata.secondary_indices) diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index c4c792e7dec..be1b31f3d20 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -235,8 +235,9 @@ public: /// additional mutation command (MATERIALIZE_TTL) will be returned. MutationCommands getMutationCommands(StorageInMemoryMetadata metadata, bool materialize_ttl, ContextPtr context, bool with_alters=false) const; - /// Check if commands have any full-text index + /// Check if commands have any full-text index or a (legacy) inverted index static bool hasFullTextIndex(const StorageInMemoryMetadata & metadata); + static bool hasLegacyInvertedIndex(const StorageInMemoryMetadata & metadata); /// Check if commands have any vector similarity index static bool hasVectorSimilarityIndex(const StorageInMemoryMetadata & metadata); diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5c993504245..0beeffcb267 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -90,6 +91,68 @@ static ColumnsStatistics getStatisticsForColumns( return all_statistics; } +/// Manages the "rows_sources" temporary file that is used during vertical merge. +class RowsSourcesTemporaryFile : public ITemporaryFileLookup +{ +public: + /// A logical name of the temporary file under which it will be known to the plan steps that use it. + static constexpr auto FILE_ID = "rows_sources"; + + explicit RowsSourcesTemporaryFile(TemporaryDataOnDiskScopePtr temporary_data_on_disk_) + : tmp_disk(std::make_unique(temporary_data_on_disk_)) + , uncompressed_write_buffer(tmp_disk->createRawStream()) + , tmp_file_name_on_disk(uncompressed_write_buffer->getFileName()) + { + } + + WriteBuffer & getTemporaryFileForWriting(const String & name) override + { + if (name != FILE_ID) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected temporary file name requested: {}", name); + + if (write_buffer) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file was already requested for writing, there musto be only one writer"); + + write_buffer = (std::make_unique(*uncompressed_write_buffer)); + return *write_buffer; + } + + std::unique_ptr getTemporaryFileForReading(const String & name) override + { + if (name != FILE_ID) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected temporary file name requested: {}", name); + + if (!finalized) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file is not finalized yet"); + + /// tmp_disk might not create real file if no data was written to it. + if (final_size == 0) + return std::make_unique(); + + /// Reopen the file for each read so that multiple reads can be performed in parallel and there is no need to seek to the beginning. + auto raw_file_read_buffer = std::make_unique(tmp_file_name_on_disk); + return std::make_unique(std::move(raw_file_read_buffer)); + } + + /// Returns written data size in bytes + size_t finalizeWriting() + { + write_buffer->finalize(); + uncompressed_write_buffer->finalize(); + finalized = true; + final_size = write_buffer->count(); + return final_size; + } + +private: + std::unique_ptr tmp_disk; + std::unique_ptr uncompressed_write_buffer; + std::unique_ptr write_buffer; + const String tmp_file_name_on_disk; + bool finalized = false; + size_t final_size = 0; +}; + static void addMissedColumnsToSerializationInfos( size_t num_rows_in_parts, const Names & part_columns, @@ -125,19 +188,19 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::extractMergingAndGatheringColu std::set key_columns(sort_key_columns_vec.cbegin(), sort_key_columns_vec.cend()); /// Force sign column for Collapsing mode - if (ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing) - key_columns.emplace(ctx->merging_params.sign_column); + if (global_ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing) + key_columns.emplace(global_ctx->merging_params.sign_column); /// Force version column for Replacing mode - if (ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing) + if (global_ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing) { - key_columns.emplace(ctx->merging_params.is_deleted_column); - key_columns.emplace(ctx->merging_params.version_column); + key_columns.emplace(global_ctx->merging_params.is_deleted_column); + key_columns.emplace(global_ctx->merging_params.version_column); } /// Force sign column for VersionedCollapsing mode. Version is already in primary key. - if (ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) - key_columns.emplace(ctx->merging_params.sign_column); + if (global_ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) + key_columns.emplace(global_ctx->merging_params.sign_column); /// Force to merge at least one column in case of empty key if (key_columns.empty()) @@ -206,7 +269,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const // E.g. `proj_a.proj` for a normal projection merge and `proj_a.tmp_proj` for a projection materialization merge. local_tmp_prefix = global_ctx->parent_part ? "" : "tmp_merge_"; } - const String local_tmp_suffix = global_ctx->parent_part ? ctx->suffix : ""; + + const String local_tmp_suffix = global_ctx->parent_part ? global_ctx->suffix : ""; if (global_ctx->merges_blocker->isCancelled() || global_ctx->merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed)) throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); @@ -231,7 +295,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const LOG_DEBUG(ctx->log, "DEDUPLICATE BY ('{}')", fmt::join(global_ctx->deduplicate_by_columns, "', '")); } - ctx->disk = global_ctx->space_reservation->getDisk(); + global_ctx->disk = global_ctx->space_reservation->getDisk(); auto local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; std::optional builder; @@ -243,7 +307,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const } else { - auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, ctx->disk, 0); + auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, global_ctx->disk, 0); builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename)); builder->withPartStorageType(global_ctx->future_part->part_format.storage_type); } @@ -364,8 +428,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const ctx->compression_codec = global_ctx->data->getCompressionCodecForPart( global_ctx->merge_list_element_ptr->total_size_bytes_compressed, global_ctx->new_data_part->ttl_infos, global_ctx->time_of_merge); - ctx->tmp_disk = std::make_unique(global_ctx->context->getTempDataOnDisk()); - switch (global_ctx->chosen_merge_algorithm) { case MergeAlgorithm::Horizontal: @@ -378,8 +440,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const } case MergeAlgorithm::Vertical: { - ctx->rows_sources_uncompressed_write_buf = ctx->tmp_disk->createRawStream(); - ctx->rows_sources_write_buf = std::make_unique(*ctx->rows_sources_uncompressed_write_buf); + ctx->rows_sources_temporary_file = std::make_shared(global_ctx->context->getTempDataOnDisk()); std::map local_merged_column_to_size; for (const auto & part : global_ctx->future_part->parts) @@ -499,11 +560,9 @@ MergeTask::StageRuntimeContextPtr MergeTask::ExecuteAndFinalizeHorizontalPart::g auto new_ctx = std::make_shared(); - new_ctx->rows_sources_write_buf = std::move(ctx->rows_sources_write_buf); - new_ctx->rows_sources_uncompressed_write_buf = std::move(ctx->rows_sources_uncompressed_write_buf); + new_ctx->rows_sources_temporary_file = std::move(ctx->rows_sources_temporary_file); new_ctx->column_sizes = std::move(ctx->column_sizes); new_ctx->compression_codec = std::move(ctx->compression_codec); - new_ctx->tmp_disk = std::move(ctx->tmp_disk); new_ctx->it_name_and_type = std::move(ctx->it_name_and_type); new_ctx->read_with_direct_io = std::move(ctx->read_with_direct_io); new_ctx->need_sync = std::move(ctx->need_sync); @@ -559,9 +618,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::prepareProjectionsToMergeAndRe const bool merge_may_reduce_rows = global_ctx->cleanup || global_ctx->deduplicate || - ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || - ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || - ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; const auto & projections = global_ctx->metadata_snapshot->getProjections(); @@ -760,11 +819,7 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const global_ctx->merge_list_element_ptr->progress.store(ctx->column_sizes->keyColumnsWeight(), std::memory_order_relaxed); /// Ensure data has written to disk. - ctx->rows_sources_write_buf->finalize(); - ctx->rows_sources_uncompressed_write_buf->finalize(); - ctx->rows_sources_uncompressed_write_buf->finalize(); - - size_t rows_sources_count = ctx->rows_sources_write_buf->count(); + size_t rows_sources_count = ctx->rows_sources_temporary_file->finalizeWriting(); /// In special case, when there is only one source part, and no rows were skipped, we may have /// skipped writing rows_sources file. Otherwise rows_sources_count must be equal to the total /// number of input rows. @@ -775,29 +830,6 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const "of bytes written to rows_sources file ({}). It is a bug.", sum_input_rows_exact, input_rows_filtered, rows_sources_count); - /// TemporaryDataOnDisk::createRawStream returns WriteBufferFromFile implementing IReadableWriteBuffer - /// and we expect to get ReadBufferFromFile here. - /// So, it's relatively safe to use dynamic_cast here and downcast to ReadBufferFromFile. - auto * wbuf_readable = dynamic_cast(ctx->rows_sources_uncompressed_write_buf.get()); - std::unique_ptr reread_buf = wbuf_readable ? wbuf_readable->tryGetReadBuffer() : nullptr; - if (!reread_buf) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot read temporary file {}", ctx->rows_sources_uncompressed_write_buf->getFileName()); - - auto * reread_buffer_raw = dynamic_cast(reread_buf.get()); - if (!reread_buffer_raw) - { - const auto & reread_buf_ref = *reread_buf; - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ReadBufferFromFileBase, but got {}", demangle(typeid(reread_buf_ref).name())); - } - /// Move ownership from std::unique_ptr to std::unique_ptr for CompressedReadBufferFromFile. - /// First, release ownership from unique_ptr to base type. - reread_buf.release(); /// NOLINT(bugprone-unused-return-value,hicpp-ignored-remove-result): we already have the pointer value in `reread_buffer_raw` - - /// Then, move ownership to unique_ptr to concrete type. - std::unique_ptr reread_buffer_from_file(reread_buffer_raw); - - /// CompressedReadBufferFromFile expects std::unique_ptr as argument. - ctx->rows_sources_read_buf = std::make_unique(std::move(reread_buffer_from_file)); ctx->it_name_and_type = global_ctx->gathering_columns.cbegin(); const auto & settings = global_ctx->context->getSettingsRef(); @@ -828,12 +860,12 @@ class ColumnGathererStep : public ITransformingStep public: ColumnGathererStep( const DataStream & input_stream_, - CompressedReadBufferFromFile * rows_sources_read_buf_, + const String & rows_sources_temporary_file_name_, UInt64 merge_block_size_rows_, UInt64 merge_block_size_bytes_, bool is_result_sparse_) : ITransformingStep(input_stream_, input_stream_.header, getTraits()) - , rows_sources_read_buf(rows_sources_read_buf_) + , rows_sources_temporary_file_name(rows_sources_temporary_file_name_) , merge_block_size_rows(merge_block_size_rows_) , merge_block_size_bytes(merge_block_size_bytes_) , is_result_sparse(is_result_sparse_) @@ -841,17 +873,20 @@ public: String getName() const override { return "ColumnGatherer"; } - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & pipeline_settings) override { const auto &header = pipeline.getHeader(); const auto input_streams_count = pipeline.getNumStreams(); - rows_sources_read_buf->seek(0, 0); + if (!pipeline_settings.temporary_file_lookup) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file lookup is not set in pipeline settings for vertical merge"); + + auto rows_sources_read_buf = pipeline_settings.temporary_file_lookup->getTemporaryFileForReading(rows_sources_temporary_file_name); auto transform = std::make_unique( header, input_streams_count, - *rows_sources_read_buf, + std::move(rows_sources_read_buf), merge_block_size_rows, merge_block_size_bytes, is_result_sparse); @@ -881,7 +916,7 @@ private: } MergeTreeData::MergingParams merging_params{}; - CompressedReadBufferFromFile * rows_sources_read_buf; + const String rows_sources_temporary_file_name; const UInt64 merge_block_size_rows; const UInt64 merge_block_size_bytes; const bool is_result_sparse; @@ -932,7 +967,7 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic const auto data_settings = global_ctx->data->getSettings(); auto merge_step = std::make_unique( merge_column_query_plan.getCurrentDataStream(), - ctx->rows_sources_read_buf.get(), //global_ctx->rows_sources_temporary_file_name, + RowsSourcesTemporaryFile::FILE_ID, data_settings->merge_max_block_size, data_settings->merge_max_block_size_bytes, is_result_sparse); @@ -961,7 +996,8 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic } auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + pipeline_settings.temporary_file_lookup = ctx->rows_sources_temporary_file; + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); auto builder = merge_column_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); return {QueryPipelineBuilder::getPipeline(std::move(*builder)), std::move(indexes_to_recalc)}; @@ -1013,10 +1049,6 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const global_ctx->to->getIndexGranularity()); ctx->column_elems_written = 0; - - /// rows_sources_read_buf is reused for each column so we need to rewind it explicitly each time - /// This sharing also prevents from from running multiple merge of individual columns in parallel. - ctx->rows_sources_read_buf->seek(0, 0); } @@ -1330,7 +1362,7 @@ public: const SortDescription & sort_description_, const Names partition_key_columns_, const MergeTreeData::MergingParams & merging_params_, - WriteBuffer * rows_sources_write_buf_, + const String & rows_sources_temporary_file_name_, UInt64 merge_block_size_rows_, UInt64 merge_block_size_bytes_, bool blocks_are_granules_size_, @@ -1340,7 +1372,7 @@ public: , sort_description(sort_description_) , partition_key_columns(partition_key_columns_) , merging_params(merging_params_) - , rows_sources_write_buf(rows_sources_write_buf_) + , rows_sources_temporary_file_name(rows_sources_temporary_file_name_) , merge_block_size_rows(merge_block_size_rows_) , merge_block_size_bytes(merge_block_size_bytes_) , blocks_are_granules_size(blocks_are_granules_size_) @@ -1350,7 +1382,7 @@ public: String getName() const override { return "MergeParts"; } - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & pipeline_settings) override { /// The order of the streams is important: when the key is matched, the elements go in the order of the source stream number. /// In the merged part, the lines with the same key must be in the ascending order of the identifier of original part, @@ -1360,6 +1392,14 @@ public: const auto &header = pipeline.getHeader(); const auto input_streams_count = pipeline.getNumStreams(); + WriteBuffer * rows_sources_write_buf = nullptr; + if (!rows_sources_temporary_file_name.empty()) + { + if (!pipeline_settings.temporary_file_lookup) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file lookup is not set in pipeline settings for vertical merge"); + rows_sources_write_buf = &pipeline_settings.temporary_file_lookup->getTemporaryFileForWriting(rows_sources_temporary_file_name); + } + switch (merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: @@ -1449,7 +1489,7 @@ private: const SortDescription sort_description; const Names partition_key_columns; const MergeTreeData::MergingParams merging_params{}; - WriteBuffer * rows_sources_write_buf; + const String rows_sources_temporary_file_name; const UInt64 merge_block_size_rows; const UInt64 merge_block_size_bytes; const bool blocks_are_granules_size; @@ -1606,8 +1646,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(sort_columns[i], 1, 1); + const bool is_vertical_merge = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); /// If merge is vertical we cannot calculate it - ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); + ctx->blocks_are_granules_size = is_vertical_merge; if (global_ctx->cleanup && !data_settings->allow_experimental_replacing_merge_with_cleanup) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed"); @@ -1616,8 +1657,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const merge_parts_query_plan.getCurrentDataStream(), sort_description, partition_key_columns, - ctx->merging_params, - ctx->rows_sources_write_buf.get(), + global_ctx->merging_params, + (is_vertical_merge ? RowsSourcesTemporaryFile::FILE_ID : ""), /// rows_sources temporaty file is used only for vertical merge data_settings->merge_max_block_size, data_settings->merge_max_block_size_bytes, ctx->blocks_are_granules_size, @@ -1682,7 +1723,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const { auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + pipeline_settings.temporary_file_lookup = ctx->rows_sources_temporary_file; + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); auto builder = merge_parts_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); global_ctx->merged_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); @@ -1726,10 +1768,10 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm } bool is_supported_storage = - ctx->merging_params.mode == MergeTreeData::MergingParams::Ordinary || - ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || - ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || - ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Ordinary || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; bool enough_ordinary_cols = global_ctx->gathering_columns.size() >= data_settings->vertical_merge_algorithm_min_columns_to_activate; diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 84f00d521fd..29b5c4452e7 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -40,6 +40,7 @@ namespace DB class MergeTask; using MergeTaskPtr = std::shared_ptr; +class RowsSourcesTemporaryFile; /** * Overview of the merge algorithm @@ -100,6 +101,7 @@ public: global_ctx->context = std::move(context_); global_ctx->holder = &holder; global_ctx->space_reservation = std::move(space_reservation_); + global_ctx->disk = global_ctx->space_reservation->getDisk(); global_ctx->deduplicate = std::move(deduplicate_); global_ctx->deduplicate_by_columns = std::move(deduplicate_by_columns_); global_ctx->cleanup = std::move(cleanup_); @@ -110,12 +112,10 @@ public: global_ctx->ttl_merges_blocker = std::move(ttl_merges_blocker_); global_ctx->txn = std::move(txn); global_ctx->need_prefix = need_prefix; + global_ctx->suffix = std::move(suffix_); + global_ctx->merging_params = std::move(merging_params_); auto prepare_stage_ctx = std::make_shared(); - - prepare_stage_ctx->suffix = std::move(suffix_); - prepare_stage_ctx->merging_params = std::move(merging_params_); - (*stages.begin())->setRuntimeContext(std::move(prepare_stage_ctx), global_ctx); } @@ -172,6 +172,7 @@ private: ContextPtr context{nullptr}; time_t time_of_merge{0}; ReservationSharedPtr space_reservation{nullptr}; + DiskPtr disk{nullptr}; bool deduplicate{false}; Names deduplicate_by_columns{}; bool cleanup{false}; @@ -210,6 +211,8 @@ private: MergeTreeTransactionPtr txn; bool need_prefix; + String suffix; + MergeTreeData::MergingParams merging_params{}; scope_guard temporary_directory_lock; UInt64 prev_elapsed_ms{0}; @@ -222,19 +225,11 @@ private: /// Proper initialization is responsibility of the author struct ExecuteAndFinalizeHorizontalPartRuntimeContext : public IStageRuntimeContext { - /// Dependencies - String suffix; - bool need_prefix; - MergeTreeData::MergingParams merging_params{}; - - TemporaryDataOnDiskPtr tmp_disk{nullptr}; - DiskPtr disk{nullptr}; bool need_remove_expired_values{false}; bool force_ttl{false}; CompressionCodecPtr compression_codec{nullptr}; size_t sum_input_rows_upper_bound{0}; - std::unique_ptr rows_sources_uncompressed_write_buf{nullptr}; - std::unique_ptr rows_sources_write_buf{nullptr}; + std::shared_ptr rows_sources_temporary_file; std::optional column_sizes{}; /// For projections to rebuild @@ -264,7 +259,6 @@ private: using ExecuteAndFinalizeHorizontalPartRuntimeContextPtr = std::shared_ptr; - struct ExecuteAndFinalizeHorizontalPart : public IStage { bool execute() override; @@ -314,11 +308,9 @@ private: struct VerticalMergeRuntimeContext : public IStageRuntimeContext { /// Begin dependencies from previous stage - std::unique_ptr rows_sources_uncompressed_write_buf{nullptr}; - std::unique_ptr rows_sources_write_buf{nullptr}; + std::shared_ptr rows_sources_temporary_file; std::optional column_sizes; CompressionCodecPtr compression_codec; - TemporaryDataOnDiskPtr tmp_disk{nullptr}; std::list::const_iterator it_name_and_type; bool read_with_direct_io{false}; bool need_sync{false}; @@ -350,13 +342,11 @@ private: size_t column_elems_written{0}; QueryPipeline column_parts_pipeline; std::unique_ptr executor; - std::unique_ptr rows_sources_read_buf{nullptr}; UInt64 elapsed_execute_ns{0}; }; using VerticalMergeRuntimeContextPtr = std::shared_ptr; - struct VerticalMergeStage : public IStage { bool execute() override; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ca619d4d208..80d61058d08 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3230,6 +3230,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental full-text index feature is not enabled (turn on setting 'allow_experimental_full_text_index')"); + if (AlterCommands::hasLegacyInvertedIndex(new_metadata) && !settings.allow_experimental_inverted_index) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Experimental inverted index feature is not enabled (turn on setting 'allow_experimental_inverted_index')"); + if (AlterCommands::hasVectorSimilarityIndex(new_metadata) && !settings.allow_experimental_vector_similarity_index) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental vector similarity index is disabled (turn on setting 'allow_experimental_vector_similarity_index')"); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 03db96dd016..6e0ae8f7cca 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,8 @@ #include #include #include +#include +#include #include #include #include @@ -74,6 +77,11 @@ namespace ErrorCodes extern const int DUPLICATED_PART_UUIDS; } +namespace FailPoints +{ + extern const char slowdown_index_analysis[]; +} + MergeTreeDataSelectExecutor::MergeTreeDataSelectExecutor(const MergeTreeData & data_) : data(data_), log(getLogger(data.getLogName() + " (SelectExecutor)")) @@ -528,6 +536,8 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( } auto query_context = context->hasQueryContext() ? context->getQueryContext() : context; + QueryStatusPtr query_status = context->getProcessListElement(); + PartFilterCounters part_filter_counters; if (query_context->getSettingsRef().allow_experimental_query_deduplication) selectPartsToReadWithUUIDFilter( @@ -549,7 +559,8 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( minmax_columns_types, partition_pruner, max_block_numbers_to_read, - part_filter_counters); + part_filter_counters, + query_status); index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::None, @@ -649,8 +660,13 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd auto mark_cache = context->getIndexMarkCache(); auto uncompressed_cache = context->getIndexUncompressedCache(); + auto query_status = context->getProcessListElement(); + auto process_part = [&](size_t part_index) { + if (query_status) + query_status->checkTimeLimit(); + auto & part = parts[part_index]; RangesInDataPart ranges(part, part_index); @@ -1545,13 +1561,22 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( const DataTypes & minmax_columns_types, const std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, - PartFilterCounters & counters) + PartFilterCounters & counters, + QueryStatusPtr query_status) { MergeTreeData::DataPartsVector prev_parts; std::swap(prev_parts, parts); for (const auto & part_or_projection : prev_parts) { + if (query_status) + query_status->checkTimeLimit(); + + fiu_do_on(FailPoints::slowdown_index_analysis, + { + sleepForMilliseconds(1000); + }); + const auto * part = part_or_projection->isProjectionPart() ? part_or_projection->getParentPart() : part_or_projection.get(); if (part_values && part_values->find(part->name) == part_values->end()) continue; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 3668eb0ad90..70536b7aa54 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -126,7 +126,8 @@ private: const DataTypes & minmax_columns_types, const std::optional & partition_pruner, const PartitionIdToMaxBlock * max_block_numbers_to_read, - PartFilterCounters & counters); + PartFilterCounters & counters, + QueryStatusPtr query_status); /// Same as previous but also skip parts uuids if any to the query context, or skip parts which uuids marked as excluded. static void selectPartsToReadWithUUIDFilter( diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 58892d0dbf2..bf9aad6545d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -29,7 +31,6 @@ namespace DB namespace ErrorCodes { - extern const int CANNOT_ALLOCATE_MEMORY; extern const int FORMAT_VERSION_TOO_OLD; extern const int ILLEGAL_COLUMN; extern const int INCORRECT_DATA; @@ -131,8 +132,7 @@ void USearchIndexWithSerialization::deserialize(ReadBuffer & istr) /// See the comment in MergeTreeIndexGranuleVectorSimilarity::deserializeBinary why we throw here throw Exception(ErrorCodes::INCORRECT_DATA, "Could not load vector similarity index. Please drop the index and create it again. Error: {}", String(result.error.release())); - if (!try_reserve(limits())) - throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for usearch index"); + try_reserve(limits()); } USearchIndexWithSerialization::Statistics USearchIndexWithSerialization::getStatistics() const @@ -270,20 +270,49 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length"); /// Reserving space is mandatory - if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + rows))) - throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index"); + size_t max_thread_pool_size = Context::getGlobalContextInstance()->getServerSettings().max_build_vector_similarity_index_thread_pool_size; + if (max_thread_pool_size == 0) + max_thread_pool_size = getNumberOfPhysicalCPUCores(); + unum::usearch::index_limits_t limits(roundUpToPowerOfTwoOrZero(index->size() + rows), max_thread_pool_size); + index->reserve(limits); - for (size_t row = 0; row < rows; ++row) + /// Vector index creation is slooooow. Add the new rows in parallel. The threadpool is global to avoid oversubscription when multiple + /// indexes are build simultaneously (e.g. multiple merges run at the same time). + auto & thread_pool = Context::getGlobalContextInstance()->getBuildVectorSimilarityIndexThreadPool(); + + auto add_vector_to_index = [&](USearchIndex::vector_key_t key, size_t row, ThreadGroupPtr thread_group) { - if (auto result = index->add(static_cast(index->size()), &column_array_data_float_data[column_array_offsets[row - 1]]); !result) + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + if (thread_group) + CurrentThread::attachToGroupIfDetached(thread_group); + + /// add is thread-safe + if (auto result = index->add(key, &column_array_data_float_data[column_array_offsets[row - 1]]); !result) + { throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release())); + } else { ProfileEvents::increment(ProfileEvents::USearchAddCount); ProfileEvents::increment(ProfileEvents::USearchAddVisitedMembers, result.visited_members); ProfileEvents::increment(ProfileEvents::USearchAddComputedDistances, result.computed_distances); } + }; + + size_t index_size = index->size(); + + for (size_t row = 0; row < rows; ++row) + { + auto key = static_cast(index_size + row); + auto task = [group = CurrentThread::getGroup(), &add_vector_to_index, key, row] { add_vector_to_index(key, row, group); }; + thread_pool.scheduleOrThrowOnError(task); } + + thread_pool.wait(); } } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 9223d6fd5b1..d81300da738 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,11 @@ namespace ErrorCodes extern const int DIRECTORY_ALREADY_EXISTS; } +namespace FailPoints +{ + extern const char stop_moving_part_before_swap_with_active[]; +} + namespace { @@ -226,6 +232,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part.temporary_directory_lock = data->getTemporaryPartDirectoryHolder(part->name); MutableDataPartStoragePtr cloned_part_storage; + bool preserve_blobs = false; if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { /// Try zero-copy replication and fallback to default copy if it's not possible @@ -253,6 +260,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me if (zero_copy_part) { /// FIXME for some reason we cannot just use this part, we have to re-create it through MergeTreeDataPartBuilder + preserve_blobs = true; zero_copy_part->is_temp = false; /// Do not remove it in dtor cloned_part_storage = zero_copy_part->getDataPartStoragePtr(); } @@ -272,7 +280,17 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part.part = std::move(builder).withPartFormatFromDisk().build(); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath()); - cloned_part.part->is_temp = data->allowRemoveStaleMovingParts(); + cloned_part.part->is_temp = false; + if (data->allowRemoveStaleMovingParts()) + { + cloned_part.part->is_temp = true; + /// Setting it in case connection to zookeeper is lost while moving + /// Otherwise part might be stuck in the moving directory due to the KEEPER_EXCEPTION in part's destructor + if (preserve_blobs) + cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::PRESERVE_BLOBS; + else + cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS; + } cloned_part.part->loadColumnsChecksumsIndexes(true, true); cloned_part.part->loadVersionMetadata(); cloned_part.part->modification_time = cloned_part.part->getDataPartStorage().getLastModified().epochTime(); @@ -282,6 +300,8 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me void MergeTreePartsMover::swapClonedPart(TemporaryClonedPart & cloned_part) const { + /// Used to get some stuck parts in the moving directory by stopping moves while pause is active + FailPointInjection::pauseFailPoint(FailPoints::stop_moving_part_before_swap_with_active); if (moves_blocker.isCancelled()) throw Exception(ErrorCodes::ABORTED, "Cancelled moving parts."); diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp index c19c2baca5c..71d89f9950a 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp @@ -1,6 +1,92 @@ -#include #include +#include +#include + +#include +#include +#include + + +namespace +{ + +size_t chooseSegmentSize( + LoggerPtr log, size_t mark_segment_size, size_t min_marks_per_task, size_t threads, size_t sum_marks, size_t number_of_replicas) +{ + /// Mark segment size determines the granularity of work distribution between replicas. + /// Namely, coordinator will take mark segments of size `mark_segment_size` granules, calculate hash of this segment and assign it to corresponding replica. + /// Small segments are good when we read a small random subset of a table, big - when we do full-scan over a large table. + /// With small segments there is a problem: consider a query like `select max(time) from wikistat`. Average size of `time` per granule is ~5KB. So when we + /// read 128 granules we still read only ~0.5MB of data. With default fs cache segment size of 4MB it means a lot of data will be downloaded and written + /// in cache for no reason. General case will look like this: + /// + /// +---------- useful data + /// v + /// +------+--+------+ + /// |------|++| | + /// |------|++| | + /// +------+--+------+ + /// ^ + /// predownloaded data -----------+ + /// + /// Having large segments solves all the problems in this case. Also bigger segments mean less requests (especially for big tables and full-scans). + /// These three values below chosen mostly intuitively. 128 granules is 1M rows - just a good starting point, 16384 seems to still make sense when reading + /// billions of rows and 1024 - is a reasonable point in between. We limit our choice to only these three options because when we change segment size + /// we essentially change distribution of data between replicas and of course we don't want to use simultaneously tens of different distributions, because + /// it would be a huge waste of cache space. + constexpr std::array borders{128, 1024, 16384}; + + LOG_DEBUG( + log, + "mark_segment_size={}, min_marks_per_task*threads={}, sum_marks/number_of_replicas^2={}", + mark_segment_size, + min_marks_per_task * threads, + sum_marks / number_of_replicas / number_of_replicas); + + /// Here we take max of two numbers: + /// * (min_marks_per_task * threads) = the number of marks we request from the coordinator each time - there is no point to have segments smaller than one unit of work for a replica + /// * (sum_marks / number_of_replicas^2) - we use consistent hashing for work distribution (including work stealing). If we have a really slow replica + /// everything except (1/number_of_replicas) portion of its work will be stolen by other replicas. And it owns (1/number_of_replicas) share of total number of marks. + /// Also important to note here that sum_marks is calculated after PK analysis, it means in particular that different segment sizes might be used for the + /// same table for different queries (it is intentional). + /// + /// Positive `mark_segment_size` means it is a user provided value, we have to preserve it. + if (mark_segment_size == 0) + mark_segment_size = std::max(min_marks_per_task * threads, sum_marks / number_of_replicas / number_of_replicas); + + /// Squeeze the value to the borders. + mark_segment_size = std::clamp(mark_segment_size, borders.front(), borders.back()); + /// After we calculated a hopefully good value for segment_size let's just find the maximal border that is not bigger than the chosen value. + for (auto border : borders | std::views::reverse) + { + if (mark_segment_size >= border) + { + LOG_DEBUG(log, "Chosen segment size: {}", border); + return border; + } + } + + UNREACHABLE(); +} + +size_t getMinMarksPerTask(size_t min_marks_per_task, const std::vector & per_part_infos) +{ + for (const auto & info : per_part_infos) + min_marks_per_task = std::max(min_marks_per_task, info->min_marks_per_task); + + if (min_marks_per_task == 0) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, "Chosen number of marks to read is zero (likely because of weird interference of settings)"); + + return min_marks_per_task; +} +} + +namespace ProfileEvents +{ +extern const Event ParallelReplicasReadMarks; +} namespace DB { @@ -36,17 +122,17 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( context_) , extension(std::move(extension_)) , coordination_mode(CoordinationMode::Default) - , min_marks_per_task(pool_settings.min_marks_for_concurrent_read) + , min_marks_per_task(getMinMarksPerTask(pool_settings.min_marks_for_concurrent_read, per_part_infos)) + , mark_segment_size(chooseSegmentSize( + log, + context_->getSettingsRef().parallel_replicas_mark_segment_size, + min_marks_per_task, + pool_settings.threads, + pool_settings.sum_marks, + extension.total_nodes_count)) { - for (const auto & info : per_part_infos) - min_marks_per_task = std::max(min_marks_per_task, info->min_marks_per_task); - - if (min_marks_per_task == 0) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, "Chosen number of marks to read is zero (likely because of weird interference of settings)"); - - extension.all_callback( - InitialAllRangesAnnouncement(coordination_mode, parts_ranges.getDescriptions(), extension.number_of_current_replica)); + extension.all_callback(InitialAllRangesAnnouncement( + coordination_mode, parts_ranges.getDescriptions(), extension.number_of_current_replica, mark_segment_size)); } MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_idx*/, MergeTreeReadTask * previous_task) @@ -111,6 +197,7 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_id if (current_task.ranges.empty()) buffered_ranges.pop_front(); + ProfileEvents::increment(ProfileEvents::ParallelReplicasReadMarks, current_sum_marks); return createTask(per_part_infos[part_idx], std::move(ranges_to_read), previous_task); } diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h index 987b7f80755..b9f2e133c4a 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h @@ -31,12 +31,13 @@ public: private: mutable std::mutex mutex; + LoggerPtr log = getLogger("MergeTreeReadPoolParallelReplicas"); const ParallelReadingExtension extension; const CoordinationMode coordination_mode; size_t min_marks_per_task{0}; + size_t mark_segment_size{0}; RangesInDataPartsDescription buffered_ranges; bool no_more_tasks_available{false}; - LoggerPtr log = getLogger("MergeTreeReadPoolParallelReplicas"); }; } diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp index f726078cfc8..13a64b4d82e 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp @@ -1,5 +1,10 @@ #include +namespace ProfileEvents +{ +extern const Event ParallelReplicasReadMarks; +} + namespace DB { @@ -50,11 +55,8 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd for (const auto & part : parts_ranges) buffered_tasks.push_back({part.data_part->info, MarkRanges{}}); - extension.all_callback(InitialAllRangesAnnouncement( - mode, - parts_ranges.getDescriptions(), - extension.number_of_current_replica - )); + extension.all_callback( + InitialAllRangesAnnouncement(mode, parts_ranges.getDescriptions(), extension.number_of_current_replica, /*mark_segment_size_=*/0)); } MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t task_idx, MergeTreeReadTask * previous_task) @@ -75,13 +77,14 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t ta { auto result = std::move(desc.ranges); desc.ranges = MarkRanges{}; + ProfileEvents::increment(ProfileEvents::ParallelReplicasReadMarks, desc.ranges.getNumberOfMarks()); return result; } } return std::nullopt; }; - if (auto result = get_from_buffer(); result) + if (auto result = get_from_buffer()) return createTask(per_part_infos[task_idx], std::move(*result), previous_task); if (no_more_tasks) @@ -104,7 +107,7 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t ta std::move(new_ranges.begin(), new_ranges.end(), std::back_inserter(old_ranges)); } - if (auto result = get_from_buffer(); result) + if (auto result = get_from_buffer()) return createTask(per_part_infos[task_idx], std::move(*result), previous_task); return nullptr; diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 7a9cebbcb2e..e20427dbff0 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -27,6 +27,7 @@ struct ParallelReadingExtension MergeTreeAllRangesCallback all_callback; MergeTreeReadTaskCallback callback; size_t number_of_current_replica{0}; + size_t total_nodes_count{0}; }; /// Base class for MergeTreeThreadSelectAlgorithm and MergeTreeSelectAlgorithm diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index f0c26c302e1..76a02bbd2c4 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -361,11 +361,23 @@ std::optional MergeTreeWhereOptimizer:: UInt64 total_size_of_moved_conditions = 0; UInt64 total_number_of_moved_columns = 0; + /// Remember positions of conditions in where_conditions list + /// to keep original order of conditions in prewhere_conditions while moving. + std::unordered_map condition_positions; + size_t position= 0; + for (const auto & condition : where_conditions) + condition_positions[&condition] = position++; + /// Move condition and all other conditions depend on the same set of columns. auto move_condition = [&](Conditions::iterator cond_it) { LOG_TRACE(log, "Condition {} moved to PREWHERE", cond_it->node.getColumnName()); - prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, cond_it); + /// Keep the original order of conditions in prewhere_conditions. + position = condition_positions[&(*cond_it)]; + auto prewhere_it = prewhere_conditions.begin(); + while (condition_positions[&(*prewhere_it)] < position && prewhere_it != prewhere_conditions.end()) + ++prewhere_it; + prewhere_conditions.splice(prewhere_it, where_conditions, cond_it); total_size_of_moved_conditions += cond_it->columns_size; total_number_of_moved_columns += cond_it->table_columns.size(); @@ -375,7 +387,12 @@ std::optional MergeTreeWhereOptimizer:: if (jt->viable && jt->columns_size == cond_it->columns_size && jt->table_columns == cond_it->table_columns) { LOG_TRACE(log, "Condition {} moved to PREWHERE", jt->node.getColumnName()); - prewhere_conditions.splice(prewhere_conditions.end(), where_conditions, jt++); + /// Keep the original order of conditions in prewhere_conditions. + position = condition_positions[&(*jt)]; + prewhere_it = prewhere_conditions.begin(); + while (condition_positions[&(*prewhere_it)] < position && prewhere_it != prewhere_conditions.end()) + ++prewhere_it; + prewhere_conditions.splice(prewhere_it, where_conditions, jt++); } else { diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index 8abf735b49f..a5730049907 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -212,14 +212,11 @@ using PartRefs = std::deque; class DefaultCoordinator : public ParallelReplicasReadingCoordinator::ImplInterface { public: - explicit DefaultCoordinator(size_t replicas_count_, size_t mark_segment_size_) + explicit DefaultCoordinator(size_t replicas_count_) : ParallelReplicasReadingCoordinator::ImplInterface(replicas_count_) - , mark_segment_size(mark_segment_size_) , replica_status(replicas_count_) , distribution_by_hash_queue(replicas_count_) { - if (mark_segment_size == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Zero value provided for `mark_segment_size`"); } ~DefaultCoordinator() override; @@ -232,7 +229,7 @@ public: private: /// This many granules will represent a single segment of marks that will be assigned to a replica - const size_t mark_segment_size{0}; + size_t mark_segment_size{0}; bool state_initialized{false}; size_t finished_replicas{0}; @@ -376,17 +373,20 @@ void DefaultCoordinator::initializeReadingState(InitialAllRangesAnnouncement ann if (state_initialized) return; - for (auto && part : announcement.description) { - auto intersecting_it = std::find_if( - all_parts_to_read.begin(), - all_parts_to_read.end(), - [&part](const Part & other) { return !other.description.info.isDisjoint(part.info); }); + /// To speedup search for adjacent parts + Parts known_parts(all_parts_to_read.begin(), all_parts_to_read.end()); - if (intersecting_it != all_parts_to_read.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Intersecting parts found in announcement"); + for (auto && part : announcement.description) + { + auto intersecting_it = known_parts.lower_bound(Part{.description = part, .replicas = {}}); - all_parts_to_read.push_back(Part{.description = std::move(part), .replicas = {announcement.replica_num}}); + if (intersecting_it != known_parts.end() && !intersecting_it->description.info.isDisjoint(part.info)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Intersecting parts found in announcement"); + + known_parts.emplace(Part{.description = part, .replicas = {}}); + all_parts_to_read.push_back(Part{.description = std::move(part), .replicas = {announcement.replica_num}}); + } } std::ranges::sort( @@ -394,7 +394,11 @@ void DefaultCoordinator::initializeReadingState(InitialAllRangesAnnouncement ann state_initialized = true; source_replica_for_parts_snapshot = announcement.replica_num; - LOG_DEBUG(log, "Reading state is fully initialized: {}", fmt::join(all_parts_to_read, "; ")); + mark_segment_size = announcement.mark_segment_size; + if (mark_segment_size == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Zero value provided for `mark_segment_size`"); + + LOG_DEBUG(log, "Reading state is fully initialized: {}, mark_segment_size: {}", fmt::join(all_parts_to_read, "; "), mark_segment_size); } void DefaultCoordinator::markReplicaAsUnavailable(size_t replica_number) @@ -869,8 +873,7 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa /// To get rid of duplicates for (auto && part: announcement.description) { - auto the_same_it = std::find_if(all_parts_to_read.begin(), all_parts_to_read.end(), - [&part] (const Part & other) { return other.description.info == part.info; }); + auto the_same_it = all_parts_to_read.find(Part{.description = part, .replicas = {}}); /// We have the same part - add the info about presence on the corresponding replica to it if (the_same_it != all_parts_to_read.end()) @@ -882,12 +885,28 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa if (state_initialized) continue; - auto covering_or_the_same_it = std::find_if(all_parts_to_read.begin(), all_parts_to_read.end(), - [&part] (const Part & other) { return other.description.info.contains(part.info) || part.info.contains(other.description.info); }); + /// Look for the first part >= current + auto covering_it = all_parts_to_read.lower_bound(Part{.description = part, .replicas = {}}); - /// It is covering part or we have covering - skip it - if (covering_or_the_same_it != all_parts_to_read.end()) - continue; + if (covering_it != all_parts_to_read.end()) + { + /// Checks if other part covers this one or this one covers the other + auto is_covered_or_covering = [&part] (const Part & other) + { + return other.description.info.contains(part.info) || part.info.contains(other.description.info); + }; + + if (is_covered_or_covering(*covering_it)) + continue; + + /// Also look at the previous part, it could be covering the current one + if (covering_it != all_parts_to_read.begin()) + { + --covering_it; + if (is_covered_or_covering(*covering_it)) + continue; + } + } new_rows_to_read += part.rows; @@ -896,6 +915,21 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa std::sort(ranges.begin(), ranges.end()); } +#ifndef NDEBUG + /// Double check that there are no intersecting parts + { + auto intersecting_part_it = std::adjacent_find(all_parts_to_read.begin(), all_parts_to_read.end(), + [] (const Part & lhs, const Part & rhs) + { + return !lhs.description.info.isDisjoint(rhs.description.info); + }); + + if (intersecting_part_it != all_parts_to_read.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Parts {} and {} intersect", + intersecting_part_it->description.info.getPartNameV1(), std::next(intersecting_part_it)->description.info.getPartNameV1()); + } +#endif + state_initialized = true; // progress_callback is not set when local plan is used for initiator @@ -1062,7 +1096,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode) switch (mode) { case CoordinationMode::Default: - pimpl = std::make_unique(replicas_count, mark_segment_size); + pimpl = std::make_unique(replicas_count); break; case CoordinationMode::WithOrder: pimpl = std::make_unique>(replicas_count); @@ -1080,8 +1114,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode) pimpl->markReplicaAsUnavailable(replica); } -ParallelReplicasReadingCoordinator::ParallelReplicasReadingCoordinator(size_t replicas_count_, size_t mark_segment_size_) - : replicas_count(replicas_count_), mark_segment_size(mark_segment_size_) +ParallelReplicasReadingCoordinator::ParallelReplicasReadingCoordinator(size_t replicas_count_) : replicas_count(replicas_count_) { } diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h index 8b463fda395..ad51d20f553 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h @@ -15,7 +15,7 @@ class ParallelReplicasReadingCoordinator public: class ImplInterface; - explicit ParallelReplicasReadingCoordinator(size_t replicas_count_, size_t mark_segment_size_ = 0); + explicit ParallelReplicasReadingCoordinator(size_t replicas_count_); ~ParallelReplicasReadingCoordinator(); void handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement); @@ -35,7 +35,6 @@ private: std::mutex mutex; const size_t replicas_count{0}; - size_t mark_segment_size{0}; std::unique_ptr pimpl; ProgressCallback progress_callback; // store the callback only to bypass it to coordinator implementation std::set replicas_used; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 6e22a3515bc..67570d78366 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -15,6 +15,7 @@ namespace ErrorCodes { extern const int SUPPORT_IS_DISABLED; extern const int REPLICA_STATUS_CHANGED; + extern const int LOGICAL_ERROR; } ReplicatedMergeTreeAttachThread::ReplicatedMergeTreeAttachThread(StorageReplicatedMergeTree & storage_) @@ -117,6 +118,67 @@ void ReplicatedMergeTreeAttachThread::checkHasReplicaMetadataInZooKeeper(const z } } +Int32 ReplicatedMergeTreeAttachThread::fixReplicaMetadataVersionIfNeeded(zkutil::ZooKeeperPtr zookeeper) +{ + const String & zookeeper_path = storage.zookeeper_path; + const String & replica_path = storage.replica_path; + const bool replica_readonly = storage.is_readonly; + + for (size_t i = 0; i != 2; ++i) + { + String replica_metadata_version_str; + const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version_str); + if (!replica_metadata_version_exists) + return -1; + + const Int32 metadata_version = parse(replica_metadata_version_str); + + if (metadata_version != 0 || replica_readonly) + { + /// No need to fix anything + return metadata_version; + } + + Coordination::Stat stat; + zookeeper->get(fs::path(zookeeper_path) / "metadata", &stat); + if (stat.version == 0) + { + /// No need to fix anything + return metadata_version; + } + + ReplicatedMergeTreeQueue & queue = storage.queue; + queue.pullLogsToQueue(zookeeper); + if (queue.getStatus().metadata_alters_in_queue != 0) + { + LOG_DEBUG(log, "No need to update metadata_version as there are ALTER_METADATA entries in the queue"); + return metadata_version; + } + + const Coordination::Requests ops = { + zkutil::makeSetRequest(fs::path(replica_path) / "metadata_version", std::to_string(stat.version), 0), + zkutil::makeCheckRequest(fs::path(zookeeper_path) / "metadata", stat.version), + }; + Coordination::Responses ops_responses; + const auto code = zookeeper->tryMulti(ops, ops_responses); + if (code == Coordination::Error::ZOK) + { + LOG_DEBUG(log, "Successfully set metadata_version to {}", stat.version); + return stat.version; + } + if (code != Coordination::Error::ZBADVERSION) + { + throw zkutil::KeeperException(code); + } + } + + /// Second attempt is only possible if metadata_version != 0 or metadata.version changed during the first attempt. + /// If metadata_version != 0, on second attempt we will return the new metadata_version. + /// If metadata.version changed, on second attempt we will either get metadata_version != 0 and return the new metadata_version or we will get metadata_alters_in_queue != 0 and return 0. + /// Either way, on second attempt this method should return. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to fix replica metadata_version in ZooKeeper after two attempts"); +} + void ReplicatedMergeTreeAttachThread::runImpl() { storage.setZooKeeper(); @@ -160,11 +222,11 @@ void ReplicatedMergeTreeAttachThread::runImpl() /// Just in case it was not removed earlier due to connection loss zookeeper->tryRemove(replica_path + "/flags/force_restore_data"); - String replica_metadata_version; - const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version); + const Int32 replica_metadata_version = fixReplicaMetadataVersionIfNeeded(zookeeper); + const bool replica_metadata_version_exists = replica_metadata_version != -1; if (replica_metadata_version_exists) { - storage.setInMemoryMetadata(metadata_snapshot->withMetadataVersion(parse(replica_metadata_version))); + storage.setInMemoryMetadata(metadata_snapshot->withMetadataVersion(replica_metadata_version)); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.h index 250a5ed34d1..bfc97442598 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.h @@ -48,6 +48,8 @@ private: void runImpl(); void finalizeInitialization(); + + Int32 fixReplicaMetadataVersionIfNeeded(zkutil::ZooKeeperPtr zookeeper); }; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index dc242a7b084..8877ebff6a1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -391,7 +391,7 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St { WriteBufferFromOwnString wb; message = PreformattedMessage::create( - "Part {} has a broken projections. It will be ignored. Broken projections info: {}", + "Part `{}` has broken projections. It will be ignored. Broken projections info: {}", part_name, getCurrentExceptionMessage(true)); LOG_DEBUG(log, message); result.action = ReplicatedCheckResult::DoNothing; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 7d8018e7577..0fa2be6a389 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -2222,6 +2222,7 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const res.inserts_in_queue = 0; res.merges_in_queue = 0; res.part_mutations_in_queue = 0; + res.metadata_alters_in_queue = 0; res.queue_oldest_time = 0; res.inserts_oldest_time = 0; res.merges_oldest_time = 0; @@ -2264,6 +2265,11 @@ ReplicatedMergeTreeQueue::Status ReplicatedMergeTreeQueue::getStatus() const res.oldest_part_to_mutate_to = entry->new_part_name; } } + + if (entry->type == LogEntry::ALTER_METADATA) + { + ++res.metadata_alters_in_queue; + } } return res; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 91a23b6a3b6..9d3349663e2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -473,6 +473,7 @@ public: UInt32 inserts_in_queue; UInt32 merges_in_queue; UInt32 part_mutations_in_queue; + UInt32 metadata_alters_in_queue; UInt32 queue_oldest_time; UInt32 inserts_oldest_time; UInt32 merges_oldest_time; diff --git a/src/Storages/MergeTree/RequestResponse.cpp b/src/Storages/MergeTree/RequestResponse.cpp index bcdeb443a0b..cd1cc2ad8e7 100644 --- a/src/Storages/MergeTree/RequestResponse.cpp +++ b/src/Storages/MergeTree/RequestResponse.cpp @@ -2,10 +2,10 @@ #include #include -#include +#include #include #include -#include +#include #include @@ -14,25 +14,29 @@ namespace DB namespace ErrorCodes { - extern const int UNKNOWN_PROTOCOL; - extern const int UNKNOWN_ELEMENT_OF_ENUM; +extern const int UNKNOWN_PROTOCOL; +extern const int UNKNOWN_ELEMENT_OF_ENUM; } namespace { - CoordinationMode validateAndGet(uint8_t candidate) - { - if (candidate <= static_cast(CoordinationMode::MAX)) - return static_cast(candidate); +CoordinationMode validateAndGet(uint8_t candidate) +{ + if (candidate <= static_cast(CoordinationMode::MAX)) + return static_cast(candidate); - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM, "Unknown reading mode: {}", candidate); - } + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM, "Unknown reading mode: {}", candidate); +} } -void ParallelReadRequest::serialize(WriteBuffer & out) const +void ParallelReadRequest::serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const { - UInt64 version = DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION; - /// Must be the first + /// Previously we didn't maintain backward compatibility and every change was breaking. + /// Particularly, we had an equality check for the version. To work around that code + /// in previous server versions we now have to lie to them about the version. + const UInt64 version = initiator_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL + ? DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION + : DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION; writeIntBinary(version, out); writeIntBinary(mode, out); @@ -53,10 +57,12 @@ ParallelReadRequest ParallelReadRequest::deserialize(ReadBuffer & in) { UInt64 version; readIntBinary(version, in); - if (version != DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION) - throw Exception(ErrorCodes::UNKNOWN_PROTOCOL, "Protocol versions for parallel reading "\ - "from replicas differ. Got: {}, supported version: {}", - version, DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION); + if (version < DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION) + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Parallel replicas protocol version is too old. Got: {}, min supported version: {}", + version, + DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION); CoordinationMode mode; size_t replica_num; @@ -70,12 +76,7 @@ ParallelReadRequest ParallelReadRequest::deserialize(ReadBuffer & in) readIntBinary(min_number_of_marks, in); description.deserialize(in); - return ParallelReadRequest( - mode, - replica_num, - min_number_of_marks, - std::move(description) - ); + return ParallelReadRequest(mode, replica_num, min_number_of_marks, std::move(description)); } void ParallelReadRequest::merge(ParallelReadRequest & other) @@ -86,9 +87,14 @@ void ParallelReadRequest::merge(ParallelReadRequest & other) description.merge(other.description); } -void ParallelReadResponse::serialize(WriteBuffer & out) const +void ParallelReadResponse::serialize(WriteBuffer & out, UInt64 replica_protocol_version) const { - UInt64 version = DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION; + /// Previously we didn't maintain backward compatibility and every change was breaking. + /// Particularly, we had an equality check for the version. To work around that code + /// in previous server versions we now have to lie to them about the version. + UInt64 version = replica_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL + ? DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION + : DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION; /// Must be the first writeIntBinary(version, out); @@ -105,25 +111,33 @@ void ParallelReadResponse::deserialize(ReadBuffer & in) { UInt64 version; readIntBinary(version, in); - if (version != DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION) - throw Exception(ErrorCodes::UNKNOWN_PROTOCOL, "Protocol versions for parallel reading " \ - "from replicas differ. Got: {}, supported version: {}", - version, DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION); + if (version < DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION) + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Parallel replicas protocol version is too old. Got: {}, min supported version: {}", + version, + DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION); readBoolText(finish, in); description.deserialize(in); } -void InitialAllRangesAnnouncement::serialize(WriteBuffer & out) const +void InitialAllRangesAnnouncement::serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const { - UInt64 version = DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION; - /// Must be the first + /// Previously we didn't maintain backward compatibility and every change was breaking. + /// Particularly, we had an equality check for the version. To work around that code + /// in previous server versions we now have to lie to them about the version. + UInt64 version = initiator_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL + ? DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION + : DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION; writeIntBinary(version, out); writeIntBinary(mode, out); description.serialize(out); writeIntBinary(replica_num, out); + if (initiator_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + writeIntBinary(mark_segment_size, out); } @@ -132,14 +146,16 @@ String InitialAllRangesAnnouncement::describe() return fmt::format("replica {}, mode {}, {}", replica_num, mode, description.describe()); } -InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffer & in) +InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffer & in, UInt64 replica_protocol_version) { UInt64 version; readIntBinary(version, in); - if (version != DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION) - throw Exception(ErrorCodes::UNKNOWN_PROTOCOL, "Protocol versions for parallel reading " \ - "from replicas differ. Got: {}, supported version: {}", - version, DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION); + if (version < DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION) + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Parallel replicas protocol version is too old. Got: {}, min supported version: {}", + version, + DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION); CoordinationMode mode; RangesInDataPartsDescription description; @@ -151,11 +167,11 @@ InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffe description.deserialize(in); readIntBinary(replica_num, in); - return InitialAllRangesAnnouncement { - mode, - description, - replica_num - }; + size_t mark_segment_size = 128; + if (replica_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + readIntBinary(mark_segment_size, in); + + return InitialAllRangesAnnouncement{mode, description, replica_num, mark_segment_size}; } } diff --git a/src/Storages/MergeTree/RequestResponse.h b/src/Storages/MergeTree/RequestResponse.h index 5f5516a6804..96b65c45bfa 100644 --- a/src/Storages/MergeTree/RequestResponse.h +++ b/src/Storages/MergeTree/RequestResponse.h @@ -63,7 +63,7 @@ struct ParallelReadRequest /// Contains only data part names without mark ranges. RangesInDataPartsDescription description; - void serialize(WriteBuffer & out) const; + void serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const; String describe() const; static ParallelReadRequest deserialize(ReadBuffer & in); void merge(ParallelReadRequest & other); @@ -78,7 +78,7 @@ struct ParallelReadResponse bool finish{false}; RangesInDataPartsDescription description; - void serialize(WriteBuffer & out) const; + void serialize(WriteBuffer & out, UInt64 replica_protocol_version) const; String describe() const; void deserialize(ReadBuffer & in); }; @@ -93,21 +93,18 @@ struct InitialAllRangesAnnouncement /// No default constructor, you must initialize all fields at once. InitialAllRangesAnnouncement( - CoordinationMode mode_, - RangesInDataPartsDescription description_, - size_t replica_num_) - : mode(mode_) - , description(description_) - , replica_num(replica_num_) + CoordinationMode mode_, RangesInDataPartsDescription description_, size_t replica_num_, size_t mark_segment_size_) + : mode(mode_), description(std::move(description_)), replica_num(replica_num_), mark_segment_size(mark_segment_size_) {} CoordinationMode mode; RangesInDataPartsDescription description; size_t replica_num; + size_t mark_segment_size; - void serialize(WriteBuffer & out) const; + void serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const; String describe(); - static InitialAllRangesAnnouncement deserialize(ReadBuffer & in); + static InitialAllRangesAnnouncement deserialize(ReadBuffer & i, UInt64 replica_protocol_version); }; diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 0365f875409..975097b5fda 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -323,7 +323,7 @@ static IMergeTreeDataPart::Checksums checkDataPart( broken_projections_message += "\n"; broken_projections_message += fmt::format( - "Part {} has a broken projection {} (error: {})", + "Part `{}` has broken projection `{}` (error: {})", data_part->name, name, exception_message); } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 18ed7df9b5d..c7ff266f30f 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -277,7 +277,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( if (has_valid_arguments) { - if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 0) + if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 0) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "It's not allowed to specify explicit zookeeper_path and replica_name " @@ -285,7 +285,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( "specify them explicitly, enable setting " "database_replicated_allow_replicated_engine_arguments."); } - else if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 1) + else if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 1) { LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "It's not recommended to explicitly specify " "zookeeper_path and replica_name in ReplicatedMergeTree arguments"); @@ -305,7 +305,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica name must be a string literal{}", verbose_help_message); - if (is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2) + if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2) { LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) " "with default arguments", zookeeper_path, replica_name); diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 9730391d429..8121f389a8d 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -24,6 +24,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int LOGICAL_ERROR; } const std::unordered_set required_configuration_keys = { @@ -146,14 +147,13 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure) { - if (engine_args.size() < 3 || engine_args.size() > (with_structure ? 8 : 7)) + if (engine_args.size() < 3 || engine_args.size() > getMaxNumberOfArguments(with_structure)) { throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage AzureBlobStorage requires 3 to {} arguments: " - "AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, " - "[account_name, account_key, format, compression, structure)])", - (with_structure ? 8 : 7)); + "Storage AzureBlobStorage requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); } for (auto & engine_arg : engine_args) @@ -272,26 +272,30 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } -void StorageAzureConfiguration::addStructureAndFormatToArgs( +void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { - if (args.size() < 3 || args.size() > 8) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage Azure requires 3 to 7 arguments: " - "StorageObjectStorage(connection_string|storage_account_url, container_name, " - "blobpath, [account_name, account_key, format, compression, structure])"); - } + if (args.size() < 3 || args.size() > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 3 to {} arguments in table function azureBlobStorage, got {}", getMaxNumberOfArguments(), args.size()); for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 4e6bfbc0745..c3adc86b124 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -22,6 +22,29 @@ public: static constexpr auto type_name = "azure"; static constexpr auto engine_name = "Azure"; + /// All possible signatures for Azure engine with structure argument (for example for azureBlobStorage table function). + static constexpr auto max_number_of_arguments_with_structure = 8; + static constexpr auto signatures_with_structure = + " - connection_string, container_name, blobpath\n" + " - connection_string, container_name, blobpath, structure \n" + " - connection_string, container_name, blobpath, format \n" + " - connection_string, container_name, blobpath, format, compression \n" + " - connection_string, container_name, blobpath, format, compression, structure \n" + " - storage_account_url, container_name, blobpath, account_name, account_key\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, structure\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n"; + + /// All possible signatures for Azure engine without structure argument (for example for AzureBlobStorage table engine). + static constexpr auto max_number_of_arguments_without_structure = 7; + static constexpr auto signatures_without_structure = + " - connection_string, container_name, blobpath\n" + " - connection_string, container_name, blobpath, format \n" + " - connection_string, container_name, blobpath, format, compression \n" + " - storage_account_url, container_name, blobpath, account_name, account_key\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n"; StorageAzureConfiguration() = default; StorageAzureConfiguration(const StorageAzureConfiguration & other); @@ -29,6 +52,9 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return engine_name; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return blob_path; } void setPath(const Path & path) override { blob_path = path; } @@ -44,7 +70,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 85eb29a3868..9b5bbdeacc1 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -24,6 +24,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int LOGICAL_ERROR; } StorageHDFSConfiguration::StorageHDFSConfiguration(const StorageHDFSConfiguration & other) @@ -83,12 +84,13 @@ StorageObjectStorage::QuerySettings StorageHDFSConfiguration::getQuerySettings(c void StorageHDFSConfiguration::fromAST(ASTs & args, ContextPtr context, bool with_structure) { - const size_t max_args_num = with_structure ? 4 : 3; - if (args.empty() || args.size() > max_args_num) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Expected not more than {} arguments", max_args_num); - } + if (args.empty() || args.size() > getMaxNumberOfArguments(with_structure)) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage HDFS requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); + std::string url_str; url_str = checkAndGetLiteralArgument(args[0], "url"); @@ -158,28 +160,34 @@ void StorageHDFSConfiguration::setURL(const std::string & url_) LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path); } -void StorageHDFSConfiguration::addStructureAndFormatToArgs( +void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { size_t count = args.size(); - if (count == 0 || count > 4) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Expected 1 to 4 arguments in table function, got {}", count); - } + if (count == 0 || count > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function hdfs, got {}", getMaxNumberOfArguments(), count); auto format_literal = std::make_shared(format_); auto structure_literal = std::make_shared(structure_); diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 04884542908..206147d7e5e 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -16,6 +16,20 @@ public: static constexpr auto type_name = "hdfs"; static constexpr auto engine_name = "HDFS"; + /// All possible signatures for HDFS engine with structure argument (for example for hdfs table function). + static constexpr auto max_number_of_arguments_with_structure = 4; + static constexpr auto signatures_with_structure = + " - uri\n" + " - uri, format\n" + " - uri, format, structure\n" + " - uri, format, structure, compression_method\n"; + + /// All possible signatures for HDFS engine without structure argument (for example for HS table engine). + static constexpr auto max_number_of_arguments_without_structure = 3; + static constexpr auto signatures_without_structure = + " - uri\n" + " - uri, format\n" + " - uri, format, compression_method\n"; StorageHDFSConfiguration() = default; StorageHDFSConfiguration(const StorageHDFSConfiguration & other); @@ -23,6 +37,9 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return engine_name; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } @@ -39,7 +56,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, diff --git a/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp index 7498e949073..e107eac0c41 100644 --- a/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp @@ -22,12 +22,12 @@ namespace DB namespace ErrorCodes { - extern const int NETWORK_ERROR; - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_SEEK_THROUGH_FILE; - extern const int SEEK_POSITION_OUT_OF_BOUND; - extern const int LOGICAL_ERROR; - extern const int UNKNOWN_FILE_SIZE; +extern const int HDFS_ERROR; +extern const int CANNOT_OPEN_FILE; +extern const int CANNOT_SEEK_THROUGH_FILE; +extern const int SEEK_POSITION_OUT_OF_BOUND; +extern const int LOGICAL_ERROR; +extern const int UNKNOWN_FILE_SIZE; } @@ -125,9 +125,12 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory max_args_num) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Expected not more than {} arguments", max_args_num); - } + if (args.empty() || args.size() > getMaxNumberOfArguments(with_structure)) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage Local requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index ba4de63ac47..84dc3855df3 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -19,6 +19,20 @@ public: using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; static constexpr auto type_name = "local"; + /// All possible signatures for Local engine with structure argument (for example for local table function). + static constexpr auto max_number_of_arguments_with_structure = 4; + static constexpr auto signatures_with_structure = + " - path\n" + " - path, format\n" + " - path, format, structure\n" + " - path, format, structure, compression_method\n"; + + /// All possible signatures for S3 engine without structure argument (for example for Local table engine). + static constexpr auto max_number_of_arguments_without_structure = 3; + static constexpr auto signatures_without_structure = + " - path\n" + " - path, format\n" + " - path, format, compression_method\n"; StorageLocalConfiguration() = default; StorageLocalConfiguration(const StorageLocalConfiguration & other) = default; @@ -26,6 +40,9 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return "Local"; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } @@ -40,7 +57,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared("/"); } - void addStructureAndFormatToArgs(ASTs &, const String &, const String &, ContextPtr) override { } + void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { } private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 7542f59dcc4..56bc6ea2f61 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -170,21 +170,20 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure) { - /// Supported signatures: S3('url') S3('url', 'format') S3('url', 'format', 'compression') S3('url', NOSIGN) S3('url', NOSIGN, 'format') S3('url', NOSIGN, 'format', 'compression') S3('url', 'aws_access_key_id', 'aws_secret_access_key') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format', 'compression') - /// with optional headers() function - size_t count = StorageURL::evalArgsAndCollectHeaders(args, headers_from_ast, context); - if (count == 0 || count > (with_structure ? 7 : 6)) + if (count == 0 || count > getMaxNumberOfArguments(with_structure)) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage S3 requires 1 to 5 arguments: " - "url, [NOSIGN | access_key_id, secret_access_key], name of used format and [compression_method]"); + "Storage S3 requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); std::unordered_map engine_args_to_idx; bool no_sign_request = false; - /// For 2 arguments we support 2 possible variants: + /// When adding new arguments in the signature don't forget to update addStructureAndFormatToArgsIfNeeded as well. + + /// For 2 arguments we support: /// - s3(source, format) /// - s3(source, NOSIGN) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not. @@ -196,10 +195,15 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ else engine_args_to_idx = {{"format", 1}}; } - /// For 3 arguments we support 2 possible variants: + /// For 3 arguments we support: + /// if with_structure == 0: + /// - s3(source, NOSIGN, format) /// - s3(source, format, compression_method) /// - s3(source, access_key_id, secret_access_key) + /// if with_structure == 1: /// - s3(source, NOSIGN, format) + /// - s3(source, format, structure) + /// - s3(source, access_key_id, secret_access_key) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or format name. else if (count == 3) { @@ -219,7 +223,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ else engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; } - /// For 4 arguments we support 3 possible variants: + /// For 4 arguments we support: /// if with_structure == 0: /// - s3(source, access_key_id, secret_access_key, session_token) /// - s3(source, access_key_id, secret_access_key, format) @@ -229,7 +233,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ /// - s3(source, access_key_id, secret_access_key, format), /// - s3(source, access_key_id, secret_access_key, session_token) /// - s3(source, NOSIGN, format, structure) - /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN or not. + /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN, format name of something else. else if (count == 4) { auto second_arg = checkAndGetLiteralArgument(args[1], "access_key_id/NOSIGN"); @@ -258,7 +262,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } } } - /// For 5 arguments we support 2 possible variants: + /// For 5 arguments we support: /// if with_structure == 0: /// - s3(source, access_key_id, secret_access_key, session_token, format) /// - s3(source, access_key_id, secret_access_key, format, compression) @@ -302,13 +306,16 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } } } + /// For 6 arguments we support: + /// if with_structure == 0: + /// - s3(source, access_key_id, secret_access_key, session_token, format, compression_method) + /// if with_structure == 1: + /// - s3(source, access_key_id, secret_access_key, format, structure, compression_method) + /// - s3(source, access_key_id, secret_access_key, session_token, format, structure) else if (count == 6) { if (with_structure) { - /// - s3(source, access_key_id, secret_access_key, format, structure, compression_method) - /// - s3(source, access_key_id, secret_access_key, session_token, format, structure) - /// We can distinguish them by looking at the 4-th argument: check if it's a format name or not auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) { @@ -324,6 +331,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"compression_method", 5}}; } } + /// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method) else if (with_structure && count == 7) { engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"structure", 5}, {"compression_method", 6}}; @@ -365,24 +373,33 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ keys = {url.key}; } -void StorageS3Configuration::addStructureAndFormatToArgs( +void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { HTTPHeaderEntries tmp_headers; size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context); - if (count == 0 || count > 6) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to 6 arguments in table function, got {}", count); + if (count == 0 || count > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function s3, got {}", getMaxNumberOfArguments(), count); auto format_literal = std::make_shared(format_); auto structure_literal = std::make_shared(structure_); @@ -394,14 +411,18 @@ void StorageS3Configuration::addStructureAndFormatToArgs( args.push_back(std::make_shared("auto")); args.push_back(structure_literal); } - /// s3(s3_url, format) or s3(s3_url, NOSIGN) + /// s3(s3_url, format) or + /// s3(s3_url, NOSIGN) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not. else if (count == 2) { auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); /// If there is NOSIGN, add format=auto before structure. if (boost::iequals(second_arg, "NOSIGN")) - args.push_back(std::make_shared("auto")); + args.push_back(format_literal); + else if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args[1] = format_literal; + args.push_back(structure_literal); } /// s3(source, format, structure) or @@ -413,21 +434,27 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(second_arg, "NOSIGN")) { + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; args.push_back(structure_literal); } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { - args[count - 1] = structure_literal; + if (second_arg == "auto") + args[1] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; } else { - /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); + /// Add format and structure arguments. + args.push_back(format_literal); args.push_back(structure_literal); } } /// s3(source, format, structure, compression_method) or /// s3(source, access_key_id, secret_access_key, format) or + /// s3(source, access_key_id, secret_access_key, session_token) or /// s3(source, NOSIGN, format, structure) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither. else if (count == 4) @@ -435,36 +462,93 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(second_arg, "NOSIGN")) { - args[count - 1] = structure_literal; + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; + if (checkAndGetLiteralArgument(args[3], "structure") == "auto") + args[3] = structure_literal; } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { - args[count - 2] = structure_literal; + if (second_arg == "auto") + args[1] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; } else { - args.push_back(structure_literal); + auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); + if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) + { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + args.push_back(structure_literal); + } + else + { + args.push_back(format_literal); + args.push_back(structure_literal); + } } } /// s3(source, access_key_id, secret_access_key, format, structure) or + /// s3(source, access_key_id, secret_access_key, session_token, format) or /// s3(source, NOSIGN, format, structure, compression_method) /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not. else if (count == 5) { - auto sedond_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); - if (boost::iequals(sedond_arg, "NOSIGN")) + auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); + if (boost::iequals(second_arg, "NOSIGN")) { - args[count - 2] = structure_literal; + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[3] = structure_literal; } else { - args[count - 1] = structure_literal; + auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); + if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) + { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; + } + else + { + if (checkAndGetLiteralArgument(args[4], "format") == "auto") + args[4] = format_literal; + args.push_back(structure_literal); + } } } - /// s3(source, access_key_id, secret_access_key, format, structure, compression) + /// s3(source, access_key_id, secret_access_key, format, structure, compression) or + /// s3(source, access_key_id, secret_access_key, session_token, format, structure) else if (count == 6) { - args[count - 2] = structure_literal; + auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); + if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) + { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; + } + else + { + if (checkAndGetLiteralArgument(args[4], "format") == "auto") + args[4] = format_literal; + if (checkAndGetLiteralArgument(args[5], "format") == "auto") + args[5] = structure_literal; + } + } + /// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method) + else if (count == 7) + { + if (checkAndGetLiteralArgument(args[4], "format") == "auto") + args[4] = format_literal; + if (checkAndGetLiteralArgument(args[5], "format") == "auto") + args[5] = structure_literal; } } } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 39a646c7df2..b36df67fb0f 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -16,6 +16,43 @@ public: static constexpr auto type_name = "s3"; static constexpr auto namespace_name = "bucket"; + /// All possible signatures for S3 storage with structure argument (for example for s3 table function). + static constexpr auto max_number_of_arguments_with_structure = 7; + static constexpr auto signatures_with_structure = + " - url\n" + " - url, NOSIGN\n" + " - url, format\n" + " - url, NOSIGN, format\n" + " - url, format, structure\n" + " - url, NOSIGN, format, structure\n" + " - url, format, structure, compression_method\n" + " - url, NOSIGN, format, structure, compression_method\n" + " - url, access_key_id, secret_access_key\n" + " - url, access_key_id, secret_access_key, session_token\n" + " - url, access_key_id, secret_access_key, format\n" + " - url, access_key_id, secret_access_key, session_token, format\n" + " - url, access_key_id, secret_access_key, format, structure\n" + " - url, access_key_id, secret_access_key, session_token, format, structure\n" + " - url, access_key_id, secret_access_key, format, structure, compression_method\n" + " - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n" + "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; + + /// All possible signatures for S3 storage without structure argument (for example for S3 table engine). + static constexpr auto max_number_of_arguments_without_structure = 6; + static constexpr auto signatures_without_structure = + " - url\n" + " - url, NOSIGN\n" + " - url, format\n" + " - url, NOSIGN, format\n" + " - url, format, compression_method\n" + " - url, NOSIGN, format, compression_method\n" + " - url, access_key_id, secret_access_key\n" + " - url, access_key_id, secret_access_key, session_token\n" + " - url, access_key_id, secret_access_key, format\n" + " - url, access_key_id, secret_access_key, session_token, format\n" + " - url, access_key_id, secret_access_key, format, compression_method\n" + " - url, access_key_id, secret_access_key, session_token, format, compression_method\n" + "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; StorageS3Configuration() = default; StorageS3Configuration(const StorageS3Configuration & other); @@ -24,6 +61,9 @@ public: std::string getEngineName() const override { return url.storage_name; } std::string getNamespaceType() const override { return namespace_name; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return url.key; } void setPath(const Path & path) override { url.key = path; } @@ -44,7 +84,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure, const String & format, diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 562ca259089..f39586c23b4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -180,7 +180,9 @@ public: virtual String getNamespace() const = 0; virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0; - virtual void addStructureAndFormatToArgs( + + /// Add/replace structure and format arguments in the AST arguments if they have 'auto' values. + virtual void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0; bool withPartitionWildcard() const; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 08a0739d929..d712e4eec20 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context); + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index 23ac92b667a..2da4aa6b665 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -33,7 +33,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int REPLICA_ALREADY_EXISTS; - extern const int INCOMPATIBLE_COLUMNS; } namespace @@ -108,8 +107,12 @@ private: } }; -ObjectStorageQueueMetadata::ObjectStorageQueueMetadata(const fs::path & zookeeper_path_, const ObjectStorageQueueSettings & settings_) +ObjectStorageQueueMetadata::ObjectStorageQueueMetadata( + const fs::path & zookeeper_path_, + const ObjectStorageQueueTableMetadata & table_metadata_, + const ObjectStorageQueueSettings & settings_) : settings(settings_) + , table_metadata(table_metadata_) , zookeeper_path(zookeeper_path_) , buckets_num(getBucketsNum(settings_)) , log(getLogger("StorageObjectStorageQueue(" + zookeeper_path_.string() + ")")) @@ -144,11 +147,6 @@ void ObjectStorageQueueMetadata::shutdown() task->deactivate(); } -void ObjectStorageQueueMetadata::checkSettings(const ObjectStorageQueueSettings & settings_) const -{ - ObjectStorageQueueTableMetadata::checkEquals(settings, settings_); -} - ObjectStorageQueueMetadata::FileStatusPtr ObjectStorageQueueMetadata::getFileStatus(const std::string & path) { return local_file_statuses->get(path, /* create */false); @@ -219,13 +217,14 @@ ObjectStorageQueueMetadata::tryAcquireBucket(const Bucket & bucket, const Proces return ObjectStorageQueueOrderedFileMetadata::tryAcquireBucket(zookeeper_path, bucket, processor, log); } -void ObjectStorageQueueMetadata::initialize( - const ConfigurationPtr & configuration, - const StorageInMemoryMetadata & storage_metadata) +void ObjectStorageQueueMetadata::syncWithKeeper( + const fs::path & zookeeper_path, + const ObjectStorageQueueTableMetadata & table_metadata, + const ObjectStorageQueueSettings & settings, + LoggerPtr log) { - const auto metadata_from_table = ObjectStorageQueueTableMetadata(*configuration, settings, storage_metadata); - const auto & columns_from_table = storage_metadata.getColumns(); const auto table_metadata_path = zookeeper_path / "metadata"; + const auto buckets_num = getBucketsNum(settings); const auto metadata_paths = settings.mode == ObjectStorageQueueMode::ORDERED ? ObjectStorageQueueOrderedFileMetadata::getMetadataPaths(buckets_num) : ObjectStorageQueueUnorderedFileMetadata::getMetadataPaths(); @@ -237,24 +236,19 @@ void ObjectStorageQueueMetadata::initialize( { if (zookeeper->exists(table_metadata_path)) { - const auto metadata_from_zk = ObjectStorageQueueTableMetadata::parse(zookeeper->get(fs::path(zookeeper_path) / "metadata")); - const auto columns_from_zk = ColumnsDescription::parse(metadata_from_zk.columns); + const auto metadata_str = zookeeper->get(fs::path(zookeeper_path) / "metadata"); + const auto metadata_from_zk = ObjectStorageQueueTableMetadata::parse(metadata_str); - metadata_from_table.checkEquals(metadata_from_zk); - if (columns_from_zk != columns_from_table) - { - throw Exception( - ErrorCodes::INCOMPATIBLE_COLUMNS, - "Table columns structure in ZooKeeper is different from local table structure. " - "Local columns:\n{}\nZookeeper columns:\n{}", - columns_from_table.toString(), columns_from_zk.toString()); - } + LOG_TRACE(log, "Metadata in keeper: {}", metadata_str); + + table_metadata.checkEquals(metadata_from_zk); return; } Coordination::Requests requests; requests.emplace_back(zkutil::makeCreateRequest(zookeeper_path, "", zkutil::CreateMode::Persistent)); - requests.emplace_back(zkutil::makeCreateRequest(table_metadata_path, metadata_from_table.toString(), zkutil::CreateMode::Persistent)); + requests.emplace_back(zkutil::makeCreateRequest( + table_metadata_path, table_metadata.toString(), zkutil::CreateMode::Persistent)); for (const auto & path : metadata_paths) { @@ -263,16 +257,27 @@ void ObjectStorageQueueMetadata::initialize( } if (!settings.last_processed_path.value.empty()) - getFileMetadata(settings.last_processed_path)->setProcessedAtStartRequests(requests, zookeeper); + { + ObjectStorageQueueOrderedFileMetadata( + zookeeper_path, + settings.last_processed_path, + std::make_shared(), + /* bucket_info */nullptr, + buckets_num, + settings.loading_retries, + log).setProcessedAtStartRequests(requests, zookeeper); + } Coordination::Responses responses; auto code = zookeeper->tryMulti(requests, responses); if (code == Coordination::Error::ZNODEEXISTS) { auto exception = zkutil::KeeperMultiException(code, requests, responses); + LOG_INFO(log, "Got code `{}` for path: {}. " "It looks like the table {} was created by another server at the same moment, " - "will retry", code, exception.getPathForFirstFailedOp(), zookeeper_path.string()); + "will retry", + code, exception.getPathForFirstFailedOp(), zookeeper_path.string()); continue; } else if (code != Coordination::Error::ZOK) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h index e5fae047ac5..71d26ca7c47 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; @@ -52,11 +53,19 @@ public: using Bucket = size_t; using Processor = std::string; - ObjectStorageQueueMetadata(const fs::path & zookeeper_path_, const ObjectStorageQueueSettings & settings_); + ObjectStorageQueueMetadata( + const fs::path & zookeeper_path_, + const ObjectStorageQueueTableMetadata & table_metadata_, + const ObjectStorageQueueSettings & settings_); + ~ObjectStorageQueueMetadata(); - void initialize(const ConfigurationPtr & configuration, const StorageInMemoryMetadata & storage_metadata); - void checkSettings(const ObjectStorageQueueSettings & settings) const; + static void syncWithKeeper( + const fs::path & zookeeper_path, + const ObjectStorageQueueTableMetadata & table_metadata, + const ObjectStorageQueueSettings & settings, + LoggerPtr log); + void shutdown(); FileMetadataPtr getFileMetadata(const std::string & path, ObjectStorageQueueOrderedFileMetadata::BucketInfoPtr bucket_info = {}); @@ -72,11 +81,17 @@ public: static size_t getBucketsNum(const ObjectStorageQueueSettings & settings); static size_t getBucketsNum(const ObjectStorageQueueTableMetadata & settings); + void checkTableMetadataEquals(const ObjectStorageQueueMetadata & other); + + const ObjectStorageQueueTableMetadata & getTableMetadata() const { return table_metadata; } + ObjectStorageQueueTableMetadata & getTableMetadata() { return table_metadata; } + private: void cleanupThreadFunc(); void cleanupThreadFuncImpl(); - const ObjectStorageQueueSettings settings; + ObjectStorageQueueSettings settings; + ObjectStorageQueueTableMetadata table_metadata; const fs::path zookeeper_path; const size_t buckets_num; @@ -89,4 +104,6 @@ private: std::shared_ptr local_file_statuses; }; +using ObjectStorageQueueMetadataPtr = std::unique_ptr; + } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp index ffae33d6f41..ba98711eff9 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp @@ -14,19 +14,23 @@ ObjectStorageQueueMetadataFactory & ObjectStorageQueueMetadataFactory::instance( return ret; } -ObjectStorageQueueMetadataFactory::FilesMetadataPtr -ObjectStorageQueueMetadataFactory::getOrCreate(const std::string & zookeeper_path, const ObjectStorageQueueSettings & settings) +ObjectStorageQueueMetadataFactory::FilesMetadataPtr ObjectStorageQueueMetadataFactory::getOrCreate( + const std::string & zookeeper_path, + ObjectStorageQueueMetadataPtr metadata) { std::lock_guard lock(mutex); auto it = metadata_by_path.find(zookeeper_path); if (it == metadata_by_path.end()) { - auto files_metadata = std::make_shared(zookeeper_path, settings); - it = metadata_by_path.emplace(zookeeper_path, std::move(files_metadata)).first; + it = metadata_by_path.emplace(zookeeper_path, std::move(metadata)).first; } else { - it->second.metadata->checkSettings(settings); + auto & metadata_from_table = metadata->getTableMetadata(); + auto & metadata_from_keeper = it->second.metadata->getTableMetadata(); + + metadata_from_table.checkEquals(metadata_from_keeper); + it->second.ref_count += 1; } return it->second.metadata; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h index a93f5ee3d83..a9975c526ef 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h @@ -13,7 +13,9 @@ public: static ObjectStorageQueueMetadataFactory & instance(); - FilesMetadataPtr getOrCreate(const std::string & zookeeper_path, const ObjectStorageQueueSettings & settings); + FilesMetadataPtr getOrCreate( + const std::string & zookeeper_path, + ObjectStorageQueueMetadataPtr metadata); void remove(const std::string & zookeeper_path); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp index cb9cdf8e186..926d5aacda4 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -32,18 +31,18 @@ namespace ObjectStorageQueueTableMetadata::ObjectStorageQueueTableMetadata( - const StorageObjectStorage::Configuration & configuration, const ObjectStorageQueueSettings & engine_settings, - const StorageInMemoryMetadata & storage_metadata) + const ColumnsDescription & columns_, + const std::string & format_) + : format_name(format_) + , columns(columns_.toString()) + , after_processing(engine_settings.after_processing.toString()) + , mode(engine_settings.mode.toString()) + , tracked_files_limit(engine_settings.tracked_files_limit) + , tracked_file_ttl_sec(engine_settings.tracked_file_ttl_sec) + , buckets(engine_settings.buckets) + , processing_threads_num(engine_settings.processing_threads_num) { - format_name = configuration.format; - after_processing = engine_settings.after_processing.toString(); - mode = engine_settings.mode.toString(); - tracked_files_limit = engine_settings.tracked_files_limit; - tracked_file_ttl_sec = engine_settings.tracked_file_ttl_sec; - buckets = engine_settings.buckets; - processing_threads_num = engine_settings.processing_threads_num; - columns = storage_metadata.getColumns().toString(); } String ObjectStorageQueueTableMetadata::toString() const @@ -65,48 +64,40 @@ String ObjectStorageQueueTableMetadata::toString() const return oss.str(); } -void ObjectStorageQueueTableMetadata::read(const String & metadata_str) +template +static auto getOrDefault( + const Poco::JSON::Object::Ptr & json, + const std::string & setting, + const std::string & compatibility_prefix, + const T & default_value) { - Poco::JSON::Parser parser; - auto json = parser.parse(metadata_str).extract(); + if (!compatibility_prefix.empty() && json->has(compatibility_prefix + setting)) + return json->getValue(compatibility_prefix + setting); - after_processing = json->getValue("after_processing"); - mode = json->getValue("mode"); + if (json->has(setting)) + return json->getValue(setting); - format_name = json->getValue("format_name"); - columns = json->getValue("columns"); + return default_value; +} - /// Check with "s3queue_" prefix for compatibility. - { - if (json->has("s3queue_tracked_files_limit")) - tracked_files_limit = json->getValue("s3queue_tracked_files_limit"); - if (json->has("s3queue_tracked_file_ttl_sec")) - tracked_file_ttl_sec = json->getValue("s3queue_tracked_file_ttl_sec"); - if (json->has("s3queue_processing_threads_num")) - processing_threads_num = json->getValue("s3queue_processing_threads_num"); - } - - if (json->has("tracked_files_limit")) - tracked_files_limit = json->getValue("tracked_files_limit"); - - if (json->has("tracked_file_ttl_sec")) - tracked_file_ttl_sec = json->getValue("tracked_file_ttl_sec"); - - if (json->has("last_processed_file")) - last_processed_path = json->getValue("last_processed_file"); - - if (json->has("processing_threads_num")) - processing_threads_num = json->getValue("processing_threads_num"); - - if (json->has("buckets")) - buckets = json->getValue("buckets"); +ObjectStorageQueueTableMetadata::ObjectStorageQueueTableMetadata(const Poco::JSON::Object::Ptr & json) + : format_name(json->getValue("format_name")) + , columns(json->getValue("columns")) + , after_processing(json->getValue("after_processing")) + , mode(json->getValue("mode")) + , tracked_files_limit(getOrDefault(json, "tracked_files_limit", "s3queue_", 0)) + , tracked_file_ttl_sec(getOrDefault(json, "tracked_files_ttl_sec", "s3queue_", 0)) + , buckets(getOrDefault(json, "buckets", "", 0)) + , processing_threads_num(getOrDefault(json, "processing_threads_num", "s3queue_", 1)) + , last_processed_path(getOrDefault(json, "last_processed_file", "s3queue_", "")) +{ } ObjectStorageQueueTableMetadata ObjectStorageQueueTableMetadata::parse(const String & metadata_str) { - ObjectStorageQueueTableMetadata metadata; - metadata.read(metadata_str); - return metadata; + Poco::JSON::Parser parser; + auto json = parser.parse(metadata_str).extract(); + return ObjectStorageQueueTableMetadata(json); } void ObjectStorageQueueTableMetadata::checkEquals(const ObjectStorageQueueTableMetadata & from_zk) const @@ -181,72 +172,17 @@ void ObjectStorageQueueTableMetadata::checkImmutableFieldsEquals(const ObjectSto ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in processing buckets. " "Stored in ZooKeeper: {}, local: {}", - ObjectStorageQueueMetadata::getBucketsNum(*this), ObjectStorageQueueMetadata::getBucketsNum(from_zk)); + ObjectStorageQueueMetadata::getBucketsNum(from_zk), ObjectStorageQueueMetadata::getBucketsNum(*this)); } } + + if (columns != from_zk.columns) + throw Exception( + ErrorCodes::METADATA_MISMATCH, + "Existing table metadata in ZooKeeper differs in columns. " + "Stored in ZooKeeper: {}, local: {}", + from_zk.columns, + columns); } -void ObjectStorageQueueTableMetadata::checkEquals(const ObjectStorageQueueSettings & current, const ObjectStorageQueueSettings & expected) -{ - if (current.after_processing != expected.after_processing) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs " - "in action after processing. Stored in ZooKeeper: {}, local: {}", - expected.after_processing.toString(), - current.after_processing.toString()); - - if (current.mode != expected.mode) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in engine mode. " - "Stored in ZooKeeper: {}, local: {}", - expected.mode.toString(), - current.mode.toString()); - - if (current.tracked_files_limit != expected.tracked_files_limit) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in max set size. " - "Stored in ZooKeeper: {}, local: {}", - expected.tracked_files_limit, - current.tracked_files_limit); - - if (current.tracked_file_ttl_sec != expected.tracked_file_ttl_sec) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in max set age. " - "Stored in ZooKeeper: {}, local: {}", - expected.tracked_file_ttl_sec, - current.tracked_file_ttl_sec); - - if (current.last_processed_path.value != expected.last_processed_path.value) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in last_processed_path. " - "Stored in ZooKeeper: {}, local: {}", - expected.last_processed_path.value, - current.last_processed_path.value); - - if (current.mode == ObjectStorageQueueMode::ORDERED) - { - if (current.buckets != expected.buckets) - { - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in buckets setting. " - "Stored in ZooKeeper: {}, local: {}", - expected.buckets, current.buckets); - } - - if (ObjectStorageQueueMetadata::getBucketsNum(current) != ObjectStorageQueueMetadata::getBucketsNum(expected)) - { - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in processing buckets. " - "Stored in ZooKeeper: {}, local: {}", - ObjectStorageQueueMetadata::getBucketsNum(current), ObjectStorageQueueMetadata::getBucketsNum(expected)); - } - } -} } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h index bbae06b66c6..d70b859ae1d 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include namespace DB @@ -16,29 +18,28 @@ class ReadBuffer; */ struct ObjectStorageQueueTableMetadata { - String format_name; - String columns; - String after_processing; - String mode; - UInt64 tracked_files_limit = 0; - UInt64 tracked_file_ttl_sec = 0; - UInt64 buckets = 0; - UInt64 processing_threads_num = 1; - String last_processed_path; + const String format_name; + const String columns; + const String after_processing; + const String mode; + const UInt64 tracked_files_limit; + const UInt64 tracked_file_ttl_sec; + const UInt64 buckets; + const UInt64 processing_threads_num; + const String last_processed_path; - ObjectStorageQueueTableMetadata() = default; ObjectStorageQueueTableMetadata( - const StorageObjectStorage::Configuration & configuration, const ObjectStorageQueueSettings & engine_settings, - const StorageInMemoryMetadata & storage_metadata); + const ColumnsDescription & columns_, + const std::string & format_); + + explicit ObjectStorageQueueTableMetadata(const Poco::JSON::Object::Ptr & json); - void read(const String & metadata_str); static ObjectStorageQueueTableMetadata parse(const String & metadata_str); String toString() const; void checkEquals(const ObjectStorageQueueTableMetadata & from_zk) const; - static void checkEquals(const ObjectStorageQueueSettings & current, const ObjectStorageQueueSettings & expected); private: void checkImmutableFieldsEquals(const ObjectStorageQueueTableMetadata & from_zk) const; diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index c1ef37e1a48..8f11836a11b 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -85,7 +85,10 @@ namespace } } - std::shared_ptr getQueueLog(const ObjectStoragePtr & storage, const ContextPtr & context, const ObjectStorageQueueSettings & table_settings) + std::shared_ptr getQueueLog( + const ObjectStoragePtr & storage, + const ContextPtr & context, + const ObjectStorageQueueSettings & table_settings) { const auto & settings = context->getSettingsRef(); switch (storage->getType()) @@ -105,7 +108,6 @@ namespace default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected object storage type: {}", storage->getType()); } - } } @@ -161,21 +163,14 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( setInMemoryMetadata(storage_metadata); LOG_INFO(log, "Using zookeeper path: {}", zk_path.string()); - task = getContext()->getSchedulePool().createTask("ObjectStorageQueueStreamingTask", [this] { threadFunc(); }); - /// Get metadata manager from ObjectStorageQueueMetadataFactory, - /// it will increase the ref count for the metadata object. - /// The ref count is decreased when StorageObjectStorageQueue::drop() method is called. - files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, *queue_settings); - try - { - files_metadata->initialize(configuration_, storage_metadata); - } - catch (...) - { - ObjectStorageQueueMetadataFactory::instance().remove(zk_path); - throw; - } + ObjectStorageQueueTableMetadata table_metadata(*queue_settings, storage_metadata.getColumns(), configuration_->format); + ObjectStorageQueueMetadata::syncWithKeeper(zk_path, table_metadata, *queue_settings, log); + + auto queue_metadata = std::make_unique(zk_path, std::move(table_metadata), *queue_settings); + files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, std::move(queue_metadata)); + + task = getContext()->getSchedulePool().createTask("ObjectStorageQueueStreamingTask", [this] { threadFunc(); }); } void StorageObjectStorageQueue::startup() diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp index 01f78673ed8..2fe1fb5905a 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp @@ -348,11 +348,10 @@ void PostgreSQLReplicationHandler::startSynchronization(bool throw_on_error) auto * materialized_storage = storage->as (); try { - auto [postgres_table_schema, postgres_table_name] = getSchemaAndTableName(table_name); - auto table_structure = fetchPostgreSQLTableStructure(tx, postgres_table_name, postgres_table_schema, true, true, true); - if (!table_structure.physical_columns) + auto table_structure = fetchTableStructure(tx, table_name); + if (!table_structure->physical_columns) throw Exception(ErrorCodes::LOGICAL_ERROR, "No columns"); - auto storage_info = StorageInfo(materialized_storage->getNested(), table_structure.physical_columns->attributes); + auto storage_info = StorageInfo(materialized_storage->getNested(), table_structure->physical_columns->attributes); nested_storages.emplace(table_name, std::move(storage_info)); } catch (Exception & e) @@ -399,9 +398,7 @@ ASTPtr PostgreSQLReplicationHandler::getCreateNestedTableQuery(StorageMaterializ postgres::Connection connection(connection_info); pqxx::nontransaction tx(connection.getRef()); - auto [postgres_table_schema, postgres_table_name] = getSchemaAndTableName(table_name); - auto table_structure = std::make_unique(fetchPostgreSQLTableStructure(tx, postgres_table_name, postgres_table_schema, true, true, true)); - + auto table_structure = fetchTableStructure(tx, table_name); auto table_override = tryGetTableOverride(current_database_name, table_name); return storage->getCreateNestedTableQuery(std::move(table_structure), table_override ? table_override->as() : nullptr); } @@ -415,16 +412,35 @@ StorageInfo PostgreSQLReplicationHandler::loadFromSnapshot(postgres::Connection std::string query_str = fmt::format("SET TRANSACTION SNAPSHOT '{}'", snapshot_name); tx->exec(query_str); - auto table_structure = fetchTableStructure(*tx, table_name); + PostgreSQLTableStructurePtr table_structure; + try + { + table_structure = fetchTableStructure(*tx, table_name); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + table_structure = std::make_unique(); + } if (!table_structure->physical_columns) throw Exception(ErrorCodes::LOGICAL_ERROR, "No table attributes"); auto table_attributes = table_structure->physical_columns->attributes; + auto columns = getTableAllowedColumns(table_name); /// Load from snapshot, which will show table state before creation of replication slot. /// Already connected to needed database, no need to add it to query. auto quoted_name = doubleQuoteWithSchema(table_name); - query_str = fmt::format("SELECT * FROM ONLY {}", quoted_name); + if (columns.empty()) + query_str = fmt::format("SELECT * FROM ONLY {}", quoted_name); + else + { + /// We should not use columns list from getTableAllowedColumns because it may have broken columns order + Strings allowed_columns; + for (const auto & column : table_structure->physical_columns->columns) + allowed_columns.push_back(column.name); + query_str = fmt::format("SELECT {} FROM ONLY {}", boost::algorithm::join(allowed_columns, ","), quoted_name); + } LOG_DEBUG(log, "Loading PostgreSQL table {}.{}", postgres_database, quoted_name); @@ -700,6 +716,37 @@ void PostgreSQLReplicationHandler::setSetting(const SettingChange & setting) } +/// Allowed columns for table from materialized_postgresql_tables_list setting +Strings PostgreSQLReplicationHandler::getTableAllowedColumns(const std::string & table_name) const +{ + Strings result; + if (tables_list.empty()) + return result; + + size_t table_pos = tables_list.find(table_name); + if (table_pos == std::string::npos) + { + return result; + } + + if (table_pos + table_name.length() + 1 > tables_list.length()) + { + return result; + } + String column_list = tables_list.substr(table_pos + table_name.length() + 1); + column_list.erase(std::remove(column_list.begin(), column_list.end(), '"'), column_list.end()); + boost::trim(column_list); + if (column_list.empty() || column_list[0] != '(') + return result; + + size_t end_bracket_pos = column_list.find(')'); + column_list = column_list.substr(1, end_bracket_pos - 1); + splitInto<','>(result, column_list); + + return result; +} + + void PostgreSQLReplicationHandler::shutdownFinal() { try @@ -749,11 +796,27 @@ std::set PostgreSQLReplicationHandler::fetchRequiredTables() Strings expected_tables; if (!tables_list.empty()) { - splitInto<','>(expected_tables, tables_list); - if (expected_tables.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse tables list: {}", tables_list); - for (auto & table_name : expected_tables) - boost::trim(table_name); + /// Removing columns `table(col1, col2)` from tables_list + String cleared_tables_list = tables_list; + while (true) + { + size_t start_bracket_pos = cleared_tables_list.find('('); + size_t end_bracket_pos = cleared_tables_list.find(')'); + if (start_bracket_pos == std::string::npos || end_bracket_pos == std::string::npos) + { + break; + } + cleared_tables_list = cleared_tables_list.substr(0, start_bracket_pos) + cleared_tables_list.substr(end_bracket_pos + 1); + } + + splitInto<','>(expected_tables, cleared_tables_list); + if (expected_tables.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse tables list: {}", tables_list); + + for (auto & table_name : expected_tables) + { + boost::trim(table_name); + } } /// Try to fetch tables list from publication if there is not tables list. @@ -864,18 +927,50 @@ std::set PostgreSQLReplicationHandler::fetchRequiredTables() /// `schema1.table1, schema2.table2, ...` -> `"schema1"."table1", "schema2"."table2", ...` /// or /// `table1, table2, ...` + setting `schema` -> `"schema"."table1", "schema"."table2", ...` + /// or + /// `table1, table2(id,name), ...` + setting `schema` -> `"schema"."table1", "schema"."table2"("id","name"), ...` if (!tables_list.empty()) { - Strings tables_names; - splitInto<','>(tables_names, tables_list); - if (tables_names.empty()) + Strings parts; + splitInto<','>(parts, tables_list); + if (parts.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Empty list of tables"); + bool is_column = false; WriteBufferFromOwnString buf; - for (auto & table_name : tables_names) + for (auto & part : parts) { - boost::trim(table_name); - buf << doubleQuoteWithSchema(table_name); + boost::trim(part); + + size_t bracket_pos = part.find('('); + if (bracket_pos != std::string::npos) + { + is_column = true; + std::string table_name = part.substr(0, bracket_pos); + boost::trim(table_name); + buf << doubleQuoteWithSchema(table_name); + + part = part.substr(bracket_pos + 1); + boost::trim(part); + buf << '('; + buf << doubleQuoteString(part); + } + else if (part.back() == ')') + { + is_column = false; + part = part.substr(0, part.size() - 1); + boost::trim(part); + buf << doubleQuoteString(part); + buf << ')'; + } + else if (is_column) + { + buf << doubleQuoteString(part); + } + else + { + buf << doubleQuoteWithSchema(part); + } buf << ","; } tables_list = buf.str(); @@ -902,23 +997,28 @@ std::set PostgreSQLReplicationHandler::fetchTablesFromPublication(pqxx:: } +template PostgreSQLTableStructurePtr PostgreSQLReplicationHandler::fetchTableStructure( - pqxx::ReplicationTransaction & tx, const std::string & table_name) const + T & tx, const std::string & table_name) const { PostgreSQLTableStructure structure; - try - { - auto [schema, table] = getSchemaAndTableName(table_name); - structure = fetchPostgreSQLTableStructure(tx, table, schema, true, true, true); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + auto [schema, table] = getSchemaAndTableName(table_name); + structure = fetchPostgreSQLTableStructure(tx, table, schema, true, true, true, getTableAllowedColumns(table_name)); return std::make_unique(std::move(structure)); } +template +PostgreSQLTableStructurePtr PostgreSQLReplicationHandler::fetchTableStructure( + pqxx::ReadTransaction & tx, const std::string & table_name) const; + +template +PostgreSQLTableStructurePtr PostgreSQLReplicationHandler::fetchTableStructure( + pqxx::ReplicationTransaction & tx, const std::string & table_name) const; + +template +PostgreSQLTableStructurePtr PostgreSQLReplicationHandler::fetchTableStructure( + pqxx::nontransaction & tx, const std::string & table_name) const; void PostgreSQLReplicationHandler::addTableToReplication(StorageMaterializedPostgreSQL * materialized_storage, const String & postgres_table_name) { diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h index 5c519053d84..8257f92ae1f 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h @@ -57,6 +57,8 @@ public: void setSetting(const SettingChange & setting); + Strings getTableAllowedColumns(const std::string & table_name) const; + void cleanupFunc(); private: @@ -94,7 +96,8 @@ private: StorageInfo loadFromSnapshot(postgres::Connection & connection, std::string & snapshot_name, const String & table_name, StorageMaterializedPostgreSQL * materialized_storage); - PostgreSQLTableStructurePtr fetchTableStructure(pqxx::ReplicationTransaction & tx, const String & table_name) const; + template + PostgreSQLTableStructurePtr fetchTableStructure(T & tx, const String & table_name) const; String doubleQuoteWithSchema(const String & table_name) const; diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index b4d83058c82..ce4950f5e7b 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -48,13 +49,15 @@ ColumnsDescription StorageSystemUsers::getColumnsDescription() {"name", std::make_shared(), "User name."}, {"id", std::make_shared(), "User ID."}, {"storage", std::make_shared(), "Path to the storage of users. Configured in the access_control_path parameter."}, - {"auth_type", std::make_shared(getAuthenticationTypeEnumValues()), - "Shows the authentication type. " + {"auth_type", std::make_shared(std::make_shared(getAuthenticationTypeEnumValues())), + "Shows the authentication types. " "There are multiple ways of user identification: " "with no password, with plain text password, with SHA256-encoded password, " "with double SHA-1-encoded password or with bcrypt-encoded password." }, - {"auth_params", std::make_shared(), "Authentication parameters in the JSON format depending on the auth_type."}, + {"auth_params", std::make_shared(std::make_shared()), + "Authentication parameters in the JSON format depending on the auth_type." + }, {"host_ip", std::make_shared(std::make_shared()), "IP addresses of hosts that are allowed to connect to the ClickHouse server." }, @@ -97,8 +100,10 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto & column_name = assert_cast(*res_columns[column_index++]); auto & column_id = assert_cast(*res_columns[column_index++]).getData(); auto & column_storage = assert_cast(*res_columns[column_index++]); - auto & column_auth_type = assert_cast(*res_columns[column_index++]).getData(); - auto & column_auth_params = assert_cast(*res_columns[column_index++]); + auto & column_auth_type = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_auth_type_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_auth_params = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_auth_params_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_names = assert_cast(assert_cast(*res_columns[column_index]).getData()); @@ -122,7 +127,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto add_row = [&](const String & name, const UUID & id, const String & storage_name, - const AuthenticationData & auth_data, + const std::vector & authentication_methods, const AllowedClientHosts & allowed_hosts, const RolesOrUsersSet & default_roles, const RolesOrUsersSet & grantees, @@ -131,11 +136,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte column_name.insertData(name.data(), name.length()); column_id.push_back(id.toUnderType()); column_storage.insertData(storage_name.data(), storage_name.length()); - column_auth_type.push_back(static_cast(auth_data.getType())); - if (auth_data.getType() == AuthenticationType::LDAP || - auth_data.getType() == AuthenticationType::KERBEROS || - auth_data.getType() == AuthenticationType::SSL_CERTIFICATE) + for (const auto & auth_data : authentication_methods) { Poco::JSON::Object auth_params_json; @@ -167,16 +169,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); Poco::JSON::Stringifier::stringify(auth_params_json, oss); - const auto str = oss.str(); + const auto authentication_params_str = oss.str(); - column_auth_params.insertData(str.data(), str.size()); - } - else - { - static constexpr std::string_view empty_json{"{}"}; - column_auth_params.insertData(empty_json.data(), empty_json.length()); + column_auth_params.insertData(authentication_params_str.data(), authentication_params_str.size()); + column_auth_type.insertValue(static_cast(auth_data.getType())); } + column_auth_params_offsets.push_back(column_auth_params.size()); + column_auth_type_offsets.push_back(column_auth_type.size()); + if (allowed_hosts.containsAnyHost()) { static constexpr std::string_view str{"::/0"}; @@ -247,7 +248,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->auth_data, user->allowed_client_hosts, + add_row(user->getName(), id, storage->getStorageName(), user->authentication_methods, user->allowed_client_hosts, user->default_roles, user->grantees, user->default_database); } } diff --git a/src/TableFunctions/ITableFunctionCluster.h b/src/TableFunctions/ITableFunctionCluster.h index 28dc43f350b..744d7139d16 100644 --- a/src/TableFunctions/ITableFunctionCluster.h +++ b/src/TableFunctions/ITableFunctionCluster.h @@ -23,7 +23,6 @@ class ITableFunctionCluster : public Base { public: String getName() const override = 0; - String getSignature() const override = 0; static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context) { @@ -46,7 +45,11 @@ protected: void parseArgumentsImpl(ASTs & args, const ContextPtr & context) override { if (args.empty()) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "The function {} should have arguments. The first argument must be the cluster name and the rest are the arguments of " + "corresponding table function", + getName()); /// Evaluate only first argument, everything else will be done Base class args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context); diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 1a58be4f75b..23e59494f61 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -57,7 +57,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & context) { - if (args.empty() || args.size() > 4) + if (args.empty() || args.size() > getMaxNumberOfArguments()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature()); for (auto & arg : args) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index ba1b7d2bb3f..4c97507b8d1 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -15,6 +15,7 @@ class Context; class ITableFunctionFileLike : public ITableFunction { public: + static constexpr auto max_number_of_arguments = 4; static constexpr auto signature = " - filename\n" " - filename, format\n" " - filename, format, structure\n" @@ -32,7 +33,7 @@ public: NameSet getVirtualsToCheckBeforeUsingStructureHint() const override; - static size_t getMaxNumberOfArguments() { return 4; } + static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &); diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 3468e5c5007..6b923f93e75 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -23,83 +23,42 @@ struct AzureDefinition { static constexpr auto name = "azureBlobStorage"; static constexpr auto storage_type_name = "Azure"; - static constexpr auto signature = " - connection_string, container_name, blobpath\n" - " - connection_string, container_name, blobpath, structure \n" - " - connection_string, container_name, blobpath, format \n" - " - connection_string, container_name, blobpath, format, compression \n" - " - connection_string, container_name, blobpath, format, compression, structure \n" - " - storage_account_url, container_name, blobpath, account_name, account_key\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, structure\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, format\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n"; - static constexpr auto max_number_of_arguments = 8; }; struct S3Definition { static constexpr auto name = "s3"; static constexpr auto storage_type_name = "S3"; - static constexpr auto signature = " - url\n" - " - url, format\n" - " - url, format, structure\n" - " - url, format, structure, compression_method\n" - " - url, access_key_id, secret_access_key\n" - " - url, access_key_id, secret_access_key, session_token\n" - " - url, access_key_id, secret_access_key, format\n" - " - url, access_key_id, secret_access_key, session_token, format\n" - " - url, access_key_id, secret_access_key, format, structure\n" - " - url, access_key_id, secret_access_key, session_token, format, structure\n" - " - url, access_key_id, secret_access_key, format, structure, compression_method\n" - " - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n" - "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; - static constexpr auto max_number_of_arguments = 8; }; struct GCSDefinition { static constexpr auto name = "gcs"; static constexpr auto storage_type_name = "GCS"; - static constexpr auto signature = S3Definition::signature; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments; }; struct COSNDefinition { static constexpr auto name = "cosn"; static constexpr auto storage_type_name = "COSN"; - static constexpr auto signature = S3Definition::signature; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments; }; struct OSSDefinition { static constexpr auto name = "oss"; static constexpr auto storage_type_name = "OSS"; - static constexpr auto signature = S3Definition::signature; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments; }; struct HDFSDefinition { static constexpr auto name = "hdfs"; static constexpr auto storage_type_name = "HDFS"; - static constexpr auto signature = " - uri\n" - " - uri, format\n" - " - uri, format, structure\n" - " - uri, format, structure, compression_method\n"; - static constexpr auto max_number_of_arguments = 4; }; struct LocalDefinition { static constexpr auto name = "local"; static constexpr auto storage_type_name = "Local"; - static constexpr auto signature = " - path\n" - " - path, format\n" - " - path, format, structure\n" - " - path, format, structure, compression_method\n"; - static constexpr auto max_number_of_arguments = 4; }; template @@ -107,14 +66,9 @@ class TableFunctionObjectStorage : public ITableFunction { public: static constexpr auto name = Definition::name; - static constexpr auto signature = Definition::signature; - - static size_t getMaxNumberOfArguments() { return Definition::max_number_of_arguments; } String getName() const override { return name; } - virtual String getSignature() const { return signature; } - bool hasStaticStructure() const override { return configuration->structure != "auto"; } bool needStructureHint() const override { return configuration->structure == "auto"; } @@ -142,7 +96,7 @@ public: const String & format, const ContextPtr & context) { - Configuration().addStructureAndFormatToArgs(args, structure, format, context); + Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context); } protected: diff --git a/src/TableFunctions/TableFunctionObjectStorageCluster.h b/src/TableFunctions/TableFunctionObjectStorageCluster.h index 296791b8bda..11e6c1fde82 100644 --- a/src/TableFunctions/TableFunctionObjectStorageCluster.h +++ b/src/TableFunctions/TableFunctionObjectStorageCluster.h @@ -19,40 +19,22 @@ struct AzureClusterDefinition { static constexpr auto name = "azureBlobStorageCluster"; static constexpr auto storage_type_name = "AzureBlobStorageCluster"; - static constexpr auto signature = " - cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]"; - static constexpr auto max_number_of_arguments = AzureDefinition::max_number_of_arguments + 1; }; struct S3ClusterDefinition { static constexpr auto name = "s3Cluster"; static constexpr auto storage_type_name = "S3Cluster"; - static constexpr auto signature = " - cluster, url\n" - " - cluster, url, format\n" - " - cluster, url, format, structure\n" - " - cluster, url, access_key_id, secret_access_key\n" - " - cluster, url, format, structure, compression_method\n" - " - cluster, url, access_key_id, secret_access_key, format\n" - " - cluster, url, access_key_id, secret_access_key, format, structure\n" - " - cluster, url, access_key_id, secret_access_key, format, structure, compression_method\n" - " - cluster, url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n" - "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments + 1; }; struct HDFSClusterDefinition { static constexpr auto name = "hdfsCluster"; static constexpr auto storage_type_name = "HDFSCluster"; - static constexpr auto signature = " - cluster_name, uri\n" - " - cluster_name, uri, format\n" - " - cluster_name, uri, format, structure\n" - " - cluster_name, uri, format, structure, compression_method\n"; - static constexpr auto max_number_of_arguments = HDFSDefinition::max_number_of_arguments + 1; }; /** -* Class implementing s3/hdfs/azureBlobStorage)Cluster(...) table functions, +* Class implementing s3/hdfs/azureBlobStorageCluster(...) table functions, * which allow to process many files from S3/HDFS/Azure blob storage on a specific cluster. * On initiator it creates a connection to _all_ nodes in cluster, discloses asterisks * in file path and dispatch each file dynamically. @@ -64,10 +46,8 @@ class TableFunctionObjectStorageCluster : public ITableFunctionCluster; diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py index 8b60f61b006..86fa1c008c9 100644 --- a/tests/ci/ci_utils.py +++ b/tests/ci/ci_utils.py @@ -1,4 +1,5 @@ import json +import logging import os import re import subprocess @@ -6,10 +7,12 @@ import sys import time from contextlib import contextmanager from pathlib import Path -from typing import Any, Iterator, List, Union, Optional, Sequence +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union import requests +logger = logging.getLogger(__name__) + class Envs: GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") @@ -36,6 +39,34 @@ def cd(path: Union[Path, str]) -> Iterator[None]: os.chdir(oldpwd) +def kill_ci_runner(message: str) -> None: + """The function to kill the current process with all parents when it's possible. + Works only when run with the set `CI` environment""" + if not os.getenv("CI", ""): # cycle import env_helper + logger.info("Running outside the CI, won't kill the runner") + return + print(f"::error::{message}") + + def get_ppid_name(pid: int) -> Tuple[int, str]: + # Avoid using psutil, it's not in stdlib + stats = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8").split() + return int(stats[3]), stats[1] + + pid = os.getpid() + pids = {} # type: Dict[str, str] + while pid: + ppid, name = get_ppid_name(pid) + pids[str(pid)] = name + pid = ppid + logger.error( + "Sleeping 5 seconds and killing all possible processes from following:\n %s", + "\n ".join(f"{p}: {n}" for p, n in pids.items()), + ) + time.sleep(5) + # The current process will be killed too + subprocess.run(f"kill -9 {' '.join(pids.keys())}", check=False, shell=True) + + class GH: class ActionsNames: RunConfig = "RunConfig" diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index 6405492cd9c..21ca1649dc9 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -19,11 +19,12 @@ from collections import defaultdict from itertools import chain from typing import Any, Dict, Optional +from ci_utils import kill_ci_runner from env_helper import IS_CI from integration_test_images import IMAGES -from tee_popen import TeePopen from report import JOB_TIMEOUT_TEST_NAME from stopwatch import Stopwatch +from tee_popen import TeePopen MAX_RETRY = 1 NUM_WORKERS = 5 @@ -332,7 +333,9 @@ class ClickhouseIntegrationTestsRunner: except subprocess.CalledProcessError as err: logging.info("docker-compose pull failed: %s", str(err)) continue - logging.error("Pulling images failed for 5 attempts. Will fail the worker.") + message = "Pulling images failed for 5 attempts. Will fail the worker." + logging.error(message) + kill_ci_runner(message) # We pass specific retcode to to ci/integration_test_check.py to skip status reporting and restart job sys.exit(13) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 75a180f356b..810bae86cb0 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -835,7 +835,9 @@ class SettingsRandomizer: ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), - "filesystem_cache_segments_batch_size": lambda: random.choice([0, 3, 10, 50]), + "filesystem_cache_segments_batch_size": lambda: random.choice( + [0, 1, 2, 3, 5, 10, 50, 100] + ), "read_from_filesystem_cache_if_exists_otherwise_bypass_cache": lambda: random.randint( 0, 1 ), diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index 2b04d843a3b..709d6641806 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -42,6 +42,7 @@ 1 1 1 + 1 diff --git a/tests/docker_scripts/stateful_runner.sh b/tests/docker_scripts/stateful_runner.sh index 86f6a299ad3..defc24781a1 100755 --- a/tests/docker_scripts/stateful_runner.sh +++ b/tests/docker_scripts/stateful_runner.sh @@ -105,7 +105,7 @@ setup_logs_replication clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql +clickhouse-client < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 039c60c8e4e..af93c93f202 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -62,7 +62,7 @@ start_server setup_logs_replication clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql +clickhouse-client < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test" diff --git a/tests/docker_scripts/stress_tests.lib b/tests/docker_scripts/stress_tests.lib index cc4c290afef..fb63ef81f80 100644 --- a/tests/docker_scripts/stress_tests.lib +++ b/tests/docker_scripts/stress_tests.lib @@ -64,6 +64,7 @@ function configure() randomize_config_boolean_value multi_read keeper_port randomize_config_boolean_value check_not_exists keeper_port randomize_config_boolean_value create_if_not_exists keeper_port + randomize_config_boolean_value remove_recursive keeper_port fi sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 8ba7b342020..97405860784 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -89,7 +89,6 @@ class Client: command = self.command[:] if stdin is None: - command += ["--multiquery"] stdin = sql else: command += ["--query", sql] diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 821bb887435..152eab461ad 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -19,6 +19,7 @@ import urllib.parse import shlex import urllib3 import requests +from pathlib import Path try: # Please, add modules that required for specific tests only here. @@ -52,6 +53,7 @@ from helpers.client import QueryRuntimeException import docker from .client import Client +from .random_settings import write_random_settings_config from .retry_decorator import retry from .config_cluster import * @@ -60,6 +62,9 @@ HELPERS_DIR = p.dirname(__file__) CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") LOCAL_DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "tests/integration/compose/") DEFAULT_ENV_NAME = ".env" +DEFAULT_BASE_CONFIG_DIR = os.environ.get( + "CLICKHOUSE_TESTS_BASE_CONFIG_DIR", "/etc/clickhouse-server/" +) SANITIZER_SIGN = "==================" @@ -444,9 +449,7 @@ class ClickHouseCluster: self.base_dir = p.dirname(base_path) self.name = name if name is not None else extract_test_name(base_path) - self.base_config_dir = base_config_dir or os.environ.get( - "CLICKHOUSE_TESTS_BASE_CONFIG_DIR", "/etc/clickhouse-server/" - ) + self.base_config_dir = base_config_dir or DEFAULT_BASE_CONFIG_DIR self.server_bin_path = p.realpath( server_bin_path or os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse") @@ -1741,6 +1744,7 @@ class ClickHouseCluster: copy_common_configs=True, config_root_name="clickhouse", extra_configs=[], + randomize_settings=True, ) -> "ClickHouseInstance": """Add an instance to the cluster. @@ -1845,6 +1849,7 @@ class ClickHouseCluster: mem_limit=mem_limit, config_root_name=config_root_name, extra_configs=extra_configs, + randomize_settings=randomize_settings, ) docker_compose_yml_dir = get_docker_compose_path() @@ -3463,6 +3468,7 @@ class ClickHouseInstance: mem_limit=None, config_root_name="clickhouse", extra_configs=[], + randomize_settings=True, ): self.name = name self.base_cmd = cluster.base_cmd @@ -3531,6 +3537,7 @@ class ClickHouseInstance: self.with_coredns = with_coredns self.coredns_config_dir = p.abspath(p.join(base_path, "coredns_config")) self.use_old_analyzer = use_old_analyzer + self.randomize_settings = randomize_settings self.main_config_name = main_config_name self.users_config_name = users_config_name @@ -4093,7 +4100,7 @@ class ClickHouseInstance: exclusion_substring="", ): if from_host: - # We check fist file exists but want to look for all rotated logs as well + # We check first file exists but want to look for all rotated logs as well result = subprocess_check_call( [ "bash", @@ -4602,6 +4609,10 @@ class ClickHouseInstance: if len(self.custom_dictionaries_paths): write_embedded_config("0_common_enable_dictionaries.xml", self.config_d_dir) + if self.randomize_settings and self.base_config_dir == DEFAULT_BASE_CONFIG_DIR: + # If custom main config is used, do not apply random settings to it + write_random_settings_config(Path(users_d_dir) / "0_random_settings.xml") + version = None version_parts = self.tag.split(".") if version_parts[0].isdigit() and version_parts[1].isdigit(): diff --git a/tests/integration/helpers/postgres_utility.py b/tests/integration/helpers/postgres_utility.py index c61c535bd62..e5cd5bdeec7 100644 --- a/tests/integration/helpers/postgres_utility.py +++ b/tests/integration/helpers/postgres_utility.py @@ -363,6 +363,7 @@ def check_tables_are_synchronized( postgres_database="postgres_database", materialized_database="test_database", schema_name="", + columns=["*"], ): assert_nested_table_is_created( instance, table_name, materialized_database, schema_name @@ -378,7 +379,7 @@ def check_tables_are_synchronized( result_query = f"select * from {table_path} order by {order_by};" expected = instance.query( - f"select * from `{postgres_database}`.`{table_name}` order by {order_by};" + f"select {','.join(columns)} from `{postgres_database}`.`{table_name}` order by {order_by};" ) result = instance.query(result_query) diff --git a/tests/integration/helpers/random_settings.py b/tests/integration/helpers/random_settings.py new file mode 100644 index 00000000000..b2319561fd7 --- /dev/null +++ b/tests/integration/helpers/random_settings.py @@ -0,0 +1,28 @@ +import random + + +def randomize_settings(): + yield "max_joined_block_size_rows", random.randint(8000, 100000) + if random.random() < 0.5: + yield "max_block_size", random.randint(8000, 100000) + + +def write_random_settings_config(destination): + with open(destination, "w") as f: + f.write( + """ + + + +""" + ) + for setting, value in randomize_settings(): + f.write(f"<{setting}>{value}\n") + + f.write( + """ + + + +""" + ) diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index c292d0cc3a4..55cf3fa36b0 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -42,9 +42,18 @@ def test_access_control_on_cluster(): ch1.query_with_retry( "CREATE USER IF NOT EXISTS Alex ON CLUSTER 'cluster'", retry_count=5 ) - assert ch1.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" - assert ch2.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" - assert ch3.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" + assert ( + ch2.query("SHOW CREATE USER Alex") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" + ) + assert ( + ch1.query("SHOW CREATE USER Alex") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" + ) + assert ( + ch3.query("SHOW CREATE USER Alex") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" + ) ch2.query_with_retry( "GRANT ON CLUSTER 'cluster' SELECT ON *.* TO Alex", retry_count=3 diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 56e007dcf5d..211a03324c4 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1236,7 +1236,10 @@ def test_system_users_required_privileges(): instance.query("GRANT SELECT ON test.* TO u2 WITH GRANT OPTION") instance.query(f"RESTORE ALL FROM {backup_name}", user="u2") - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 DEFAULT ROLE r1\n" + assert ( + instance.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH no_password DEFAULT ROLE r1\n" + ) assert instance.query("SHOW GRANTS FOR u1") == TSV( ["GRANT SELECT ON test.* TO u1", "GRANT r1 TO u1"] ) diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index d20e10e8a04..6d578f37bf5 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -769,7 +769,8 @@ def test_system_users(): ) assert ( - node1.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS custom_a = 123\n" + node1.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS custom_a = 123\n" ) assert node1.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON default.tbl TO u1\n" diff --git a/tests/integration/test_backward_compatibility/configs/clusters.xml b/tests/integration/test_backward_compatibility/configs/clusters.xml new file mode 100644 index 00000000000..ac773152df9 --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/clusters.xml @@ -0,0 +1,20 @@ + + + + + + node0 + 9000 + + + node1 + 9000 + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py b/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py new file mode 100644 index 00000000000..e1b9049ef5d --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py @@ -0,0 +1,64 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +cluster_name = "parallel_replicas" +nodes = [ + cluster.add_instance( + f"node{num}", + main_configs=["configs/clusters.xml"], + with_zookeeper=False, + image="clickhouse/clickhouse-server", + tag="23.11", # earlier versions lead to "Not found column sum(a) in block." exception 🤷 + stay_alive=True, + use_old_analyzer=True, + with_installed_binary=True, + ) + for num in range(2) +] + [ + cluster.add_instance( + "node2", + main_configs=["configs/clusters.xml"], + with_zookeeper=False, + use_old_analyzer=True, + ) +] + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_backward_compatability(start_cluster): + for node in nodes: + node.query("create table t (a UInt64) engine = MergeTree order by tuple()") + node.query("insert into t select number % 100000 from numbers_mt(1000000)") + + # all we want is the query to run without errors + for node in nodes: + assert ( + node.query( + """ + select sum(a) + from t + """, + settings={ + "cluster_for_parallel_replicas": "parallel_replicas", + "max_parallel_replicas": 3, + "allow_experimental_parallel_reading_from_replicas": 1, + "parallel_replicas_for_non_replicated_merge_tree": 1, + }, + ) + == "49999500000\n" + ) + + for node in nodes: + node.query("drop table t") diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index 578ff42369c..76cbdbf9e57 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -405,7 +405,7 @@ def test_materialize_broken_projection(cluster): assert "NO_FILE_IN_DATA_PART" in get_broken_projections_info( node, table_name, part="all_1_1_0", projection="proj1" ) - assert "Part all_1_1_0 has a broken projection proj1" in check_table_full( + assert "Part `all_1_1_0` has broken projection `proj1`" in check_table_full( node, table_name ) @@ -415,13 +415,13 @@ def test_materialize_broken_projection(cluster): assert "FILE_DOESNT_EXIST" in get_broken_projections_info( node, table_name, part="all_1_1_0", projection="proj2" ) - assert "Part all_1_1_0 has a broken projection proj2" in check_table_full( + assert "Part `all_1_1_0` has broken projection `proj2`" in check_table_full( node, table_name ) materialize_projection(node, table_name, "proj1") - assert "has a broken projection" not in check_table_full(node, table_name) + assert "has broken projection" not in check_table_full(node, table_name) def test_broken_ignored_replicated(cluster): @@ -443,13 +443,13 @@ def test_broken_ignored_replicated(cluster): check(node, table_name2, 1) break_projection(node, table_name, "proj1", "all_0_0_0", "data") - assert "Part all_0_0_0 has a broken projection proj1" in check_table_full( + assert "Part `all_0_0_0` has broken projection `proj1`" in check_table_full( node, table_name ) break_part(node, table_name, "all_0_0_0") node.query(f"SYSTEM SYNC REPLICA {table_name}") - assert "has a broken projection" not in check_table_full(node, table_name) + assert "has broken projection" not in check_table_full(node, table_name) def get_random_string(string_length=8): @@ -528,7 +528,7 @@ def test_broken_projections_in_backups_3(cluster): check(node, table_name, 1) break_projection(node, table_name, "proj1", "all_1_1_0", "part") - assert "Part all_1_1_0 has a broken projection proj1" in check_table_full( + assert "Part `all_1_1_0` has broken projection `proj1`" in check_table_full( node, table_name ) assert "FILE_DOESNT_EXIST" in get_broken_projections_info( @@ -735,11 +735,14 @@ def test_mutation_with_broken_projection(cluster): f"ALTER TABLE {table_name} DELETE WHERE _part == 'all_0_0_0_4' SETTINGS mutations_sync = 1" ) + parts = get_parts(node, table_name) # All parts changes because this is how alter delete works, # but all parts apart from the first have only hardlinks to files in previous part. - assert ["all_0_0_0_5", "all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts( - node, table_name - ) or ["all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts(node, table_name) + assert ["all_0_0_0_5", "all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == parts or [ + "all_1_1_0_5", + "all_2_2_0_5", + "all_3_3_0_5", + ] == parts # Still broken because it was hardlinked. broken = get_broken_projections_info(node, table_name) @@ -752,11 +755,13 @@ def test_mutation_with_broken_projection(cluster): f"ALTER TABLE {table_name} DELETE WHERE c == 13 SETTINGS mutations_sync = 1" ) - assert ["all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == get_parts( - node, table_name - ) or ["all_0_0_0_6", "all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == get_parts( - node, table_name - ) + parts = get_parts(node, table_name) + assert ["all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == parts or [ + "all_0_0_0_6", + "all_1_1_0_6", + "all_2_2_0_6", + "all_3_3_0_6", + ] == parts # Not broken anymore. assert not get_broken_projections_info(node, table_name) diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index a710295505e..ae756ef1e27 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -46,7 +46,7 @@ def test_create(): def check(): assert ( instance.query("SHOW CREATE USER u1") - == "CREATE USER u1 SETTINGS PROFILE `s1`\n" + == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS PROFILE `s1`\n" ) assert ( instance.query("SHOW CREATE USER u2") @@ -99,7 +99,7 @@ def test_alter(): def check(): assert ( instance.query("SHOW CREATE USER u1") - == "CREATE USER u1 SETTINGS PROFILE `s1`\n" + == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS PROFILE `s1`\n" ) assert ( instance.query("SHOW CREATE USER u2") @@ -147,7 +147,10 @@ def test_drop(): instance.query("DROP SETTINGS PROFILE s1") def check(): - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" + assert ( + instance.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH no_password\n" + ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE `s2`\n" diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index 0b8c1771a40..1cf05ff9df4 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -18,12 +18,16 @@ def started_cluster(): def test_enabling_access_management(): + instance.query("DROP USER IF EXISTS Alex") + instance.query("CREATE USER Alex", user="default") assert ( - instance.query("SHOW CREATE USER Alex", user="default") == "CREATE USER Alex\n" + instance.query("SHOW CREATE USER Alex", user="default") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" ) assert ( - instance.query("SHOW CREATE USER Alex", user="readonly") == "CREATE USER Alex\n" + instance.query("SHOW CREATE USER Alex", user="readonly") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" ) assert "Not enough privileges" in instance.query_and_get_error( "SHOW CREATE USER Alex", user="xyz" @@ -35,3 +39,5 @@ def test_enabling_access_management(): assert "Not enough privileges" in instance.query_and_get_error( "CREATE USER Robin", user="xyz" ) + + instance.query("DROP USER IF EXISTS Alex") diff --git a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py index ad3d35d8bbd..50bf0f26075 100644 --- a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py +++ b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py @@ -36,7 +36,8 @@ def cleanup_after_test(): yield finally: instance.query("DROP USER IF EXISTS A, B, C") - instance.query("DROP TABLE IF EXISTS test.view_1") + + instance.query("DROP TABLE IF EXISTS test.view_1, test.view_2, default.table") def test_smoke(): @@ -144,7 +145,8 @@ def test_allowed_grantees(): instance.query("ALTER USER A GRANTEES ANY EXCEPT B") assert ( - instance.query("SHOW CREATE USER A") == "CREATE USER A GRANTEES ANY EXCEPT B\n" + instance.query("SHOW CREATE USER A") + == "CREATE USER A IDENTIFIED WITH no_password GRANTEES ANY EXCEPT B\n" ) expected_error = "user `B` is not allowed as grantee" assert expected_error in instance.query_and_get_error( @@ -157,7 +159,10 @@ def test_allowed_grantees(): instance.query("REVOKE SELECT ON test.table FROM B", user="A") instance.query("ALTER USER A GRANTEES ANY") - assert instance.query("SHOW CREATE USER A") == "CREATE USER A\n" + assert ( + instance.query("SHOW CREATE USER A") + == "CREATE USER A IDENTIFIED WITH no_password\n" + ) instance.query("GRANT SELECT ON test.table TO B", user="A") assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" @@ -169,7 +174,8 @@ def test_allowed_grantees(): instance.query("CREATE USER C GRANTEES ANY EXCEPT C") assert ( - instance.query("SHOW CREATE USER C") == "CREATE USER C GRANTEES ANY EXCEPT C\n" + instance.query("SHOW CREATE USER C") + == "CREATE USER C IDENTIFIED WITH no_password GRANTEES ANY EXCEPT C\n" ) instance.query("GRANT SELECT ON test.table TO C WITH GRANT OPTION") assert instance.query("SELECT * FROM test.table", user="C") == "1\t5\n2\t10\n" @@ -387,15 +393,22 @@ def test_introspection(): instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) - assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"]) - assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"]) + assert instance.query("SHOW CREATE USERS A") == TSV( + ["CREATE USER A IDENTIFIED WITH no_password"] + ) + assert instance.query("SHOW CREATE USERS B") == TSV( + ["CREATE USER B IDENTIFIED WITH no_password"] + ) assert instance.query("SHOW CREATE USERS A,B") == TSV( - ["CREATE USER A", "CREATE USER B"] + [ + "CREATE USER A IDENTIFIED WITH no_password", + "CREATE USER B IDENTIFIED WITH no_password", + ] ) assert instance.query("SHOW CREATE USERS") == TSV( [ - "CREATE USER A", - "CREATE USER B", + "CREATE USER A IDENTIFIED WITH no_password", + "CREATE USER B IDENTIFIED WITH no_password", "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE `default`", ] ) @@ -454,8 +467,8 @@ def test_introspection(): assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user="A") expected_access1 = ( - "CREATE USER A\n" - "CREATE USER B\n" + "CREATE USER A IDENTIFIED WITH no_password\n" + "CREATE USER B IDENTIFIED WITH no_password\n" "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE `default`" ) expected_access2 = ( @@ -473,8 +486,8 @@ def test_introspection(): [ "A", "local_directory", - "no_password", - "{}", + "['no_password']", + "['{}']", "['::/0']", "[]", "[]", @@ -486,8 +499,8 @@ def test_introspection(): [ "B", "local_directory", - "no_password", - "{}", + "['no_password']", + "['{}']", "['::/0']", "[]", "[]", diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt new file mode 100644 index 00000000000..cabc53fa809 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPDCCAiQCFBXNOvsLA+dqmX/TkYG9JXdD5m72MA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMCmNsaWNraG91c2UwIBcNMjIw +NDIxMTAzNDU1WhgPMjEyMjAzMjgxMDM0NTVaMFkxCzAJBgNVBAYTAkFVMRMwEQYD +VQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM +dGQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAKaXz596N4NC2zZdIqdwZbSYAtNdBCsBVPt5YT9F640aF5zOogPZyxGP +ENyOZwABi/7HhwFbH657xyRvi8lTau8dZL+0tbakyoIn1Tw6j+/3GXTjLduJSy6C +mOf4OzsrFC8mYgU+7p5ijvWVlO9h5NMbLdAPSIB5WSHhmSORH5LgjoK6oMOYdRod +GmfHqSbwPVwy3Li5SXlniCQmJsM0zl64LFbJ/NU+13qETmhBiDgmh0Svi+wzSzqZ +q1PIX92T3k44IXNZbvF7lKbUOS9Xb3BoxA4cDqRcTx4x73xRDwodSmqiuQOC99HI +A0C/tZJ25VNAGjLKukPSHqYscq2PAsUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA +IDQwjf/ja3TfOXrz+Gn1eErSKnWS3asjRT9rYWQsy3tzVUkMIcszrG+FqTR16g5H +ZWyuEOi6KIRmda3SYKdLKmtQLrgx6/d/jvH5TQ0LTFZrp6vh0lo3pV+L6fLo1ZRD +V1i8jW/7HHNyqJamUXOjwA0DpPOMkdtwuyV+rJ+2bTG1ZSK33O4Ae2CY5+dad6zy +YI6b1c9flWfDznuNEMH7jDDjKgXwjZGeU53FiuuhHiNyRchsr/B9eIBsom8oykiD +kch4cnAxx2E+m3hLYzupkXHOVQ5CNpVk8PGUCIGcyqDxPt+fOj1WbDQ9laEcfhmV +kR+vHmzOqWZnHU4QeMqDig== +-----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key new file mode 100644 index 00000000000..1e29a4c8fa1 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,4E14FF586022476CD22AAFB662BB0E40 + +dpJZKq5k+fMuC7XECfTSRjPeOEl9wNuVtZkcjEWaHN8ky4umND7ARyRyuU1Nk7cy +fpCFlFKOqDfCkT5zVK/fB6pF32wqAI7sqeSuYPfQY0+L77yRdwM6L46WslzVKZYE +lXD1AmqKT/LgF3+eBY5slkAAJo10zYDgKEwnoQVBp31YW2/+6oAGaY/O6x3p7aTG +dw9CP+SFc0o8lPl1lsSovdNXDUiVCftvClog7hwyDv8AhHyGgynw3UJXX8UlyWu+ +Zz5zpgrvB2gvDLeoZZ6qjMGvtpEwlYBh4de9ZOsvQUpXEEfkQFtJV0j613OCQune +LoTxKpYV1V/mZX4HPaJ1oC0OJ7XcliAOSS9K49YobTuz7Kg5Wg3bVGo9xRfYDjch +rVeMP4u5RXSGuHL23FPMfK6EqcldrFyRwLaY/IV1Yl6UNUMKAphn/WMtWVuT3TiT +QMCI2VRt7ItwZwwFn5RgyDweWdFf5v3AmN/lOhATDBqosahkPxDZ1VZ6OBPoJLPM +UrDWH/lqrByeEjtAOwr5UsWKwLuJ8qUxQ4TchHwFKOwy6VsrRwMQ3ZWi2govPF9I +W0sfLj5Ulfjx6zHdqnF48a1Elit4JH6inBEWFuj7bmlOotq+PHoeT61zAwW+gnrG +3JTo3XnaE2WwRDpqvKYHWLv/J218rq8PtIaq9gjr55odPfIt8lkJ1XzF4WQ21rIJ +GNWZ3xz4fxpvrKnQyAKGu0ZcdjA1nqs16oiVr+UnJoXmkM5yBCic4fZYwPTSQHYS +ZxwaTzEjfeGxrSeLrN9CgoweucvogOvUjJOBcW/py80du8vWz0YyzMhg3o0YeGME +C+Kts/YWxmyfw4DaWt8RtWCKl85hEmz8RODvkMLGtLzvVoSyLQWqp1NhGIlFtzXs +7sPLulUeyD2avTC/RB/Pu9Nk80c0368BxCoeYbiFWZpaN70SJmCUE5H59J2d0olw +5v2RVjLBi8wqnzoa0+2L8wnG7IQGadS97dj0eBR+JXNtoJhVrurS80RJ6B0bNxdu +gX8otfnJYsZyK5hbEhcQqLdnyGhDEE8YHe7Hv9stWwLAFOfOMzyzC06lFS1eNiw4 +FJyXJUhDieb8EqetouAC8dNVXz4Q1zOTlGuAbGoKm5v0U5IhCQap9GUSW5QiUgOQ +AEMs9aGfd91R+IcDf19mZptsQLYA6MGBN6fm+3O2iZImKIbF+ZZo0S6liFFmn6lm +M+diTzaoiqgEkiXOuRhdQUMaiGV8BMZxv8qUH6/vyC3gSueoTio0f9PfASDYfvXD +A3GuI87P6LF1it2UlN6ssFoXTZdfQQZwRmNuqOqw+BJOJHrR6trcXOCZOQ77Qnvd +M5a348gIzluVUkExAPGCsySQWMx4Of5NBF28jEC3+TAwkRqBV2ZHmfGLWnvwaB+A +YUeKtpWblfG1lsrDAdwL2dilU95oby+35sExX7M2dCrL9Y2P5oTCW3u12//ZSLeL +Yhi1Rzol6LAuesZCVF0Zv/YYDhzAckJfT/qXK5B5pz9saswxCUBEpiKlLpVsjOFJ +2bHm8NgOMD5b3cdh1kvts4wZe+giry7LHsn46f+9VqN+gA6XxeVsPyb4uO1KW3SN +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/server.crt b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt similarity index 100% rename from tests/integration/test_keeper_internal_secure/configs/server.crt rename to tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt diff --git a/tests/integration/test_keeper_internal_secure/configs/server.key b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key similarity index 100% rename from tests/integration/test_keeper_internal_secure/configs/server.key rename to tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml index 986b503ebe3..dabf280bc36 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml @@ -1,5 +1,6 @@ + 0 9181 1 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml index 652b1992f46..21d8f1e0eb3 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml @@ -1,5 +1,6 @@ + 0 9181 2 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml index 6507f97473b..5d8cfb8b3e7 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml @@ -1,5 +1,6 @@ + 0 9181 3 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml deleted file mode 100644 index 37eb2624b1b..00000000000 --- a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - /etc/clickhouse-server/config.d/server.crt - /etc/clickhouse-server/config.d/server.key - /etc/clickhouse-server/config.d/rootCA.pem - true - none - true - sslv2,sslv3 - true - - - diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml new file mode 100644 index 00000000000..a444122b09d --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml @@ -0,0 +1,10 @@ +openSSL: + server: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml new file mode 100644 index 00000000000..51b65c5253a --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml @@ -0,0 +1,14 @@ +openSSL: + server: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + privateKeyPassphraseHandler: + name: KeyFileHandler + options: + password: 'PASSWORD' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index 2d45e95e4ff..8cab03b6e2d 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -2,44 +2,55 @@ import pytest from helpers.cluster import ClickHouseCluster -import random -import string +import helpers.keeper_utils as ku +from multiprocessing.dummy import Pool import os -import time +CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance( - "node1", - main_configs=[ - "configs/enable_secure_keeper1.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) -node2 = cluster.add_instance( - "node2", - main_configs=[ - "configs/enable_secure_keeper2.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) -node3 = cluster.add_instance( - "node3", - main_configs=[ - "configs/enable_secure_keeper3.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) +nodes = [ + cluster.add_instance( + "node1", + main_configs=[ + "configs/enable_secure_keeper1.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), + cluster.add_instance( + "node2", + main_configs=[ + "configs/enable_secure_keeper2.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), + cluster.add_instance( + "node3", + main_configs=[ + "configs/enable_secure_keeper3.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), +] -from kazoo.client import KazooClient, KazooState +from kazoo.client import KazooClient @pytest.fixture(scope="module") @@ -61,23 +72,113 @@ def get_fake_zk(nodename, timeout=30.0): return _fake_zk_instance -def test_secure_raft_works(started_cluster): +def run_test(): + node_zks = [] try: - node1_zk = get_fake_zk("node1") - node2_zk = get_fake_zk("node2") - node3_zk = get_fake_zk("node3") + for node in nodes: + node_zks.append(get_fake_zk(node.name)) - node1_zk.create("/test_node", b"somedata1") - node2_zk.sync("/test_node") - node3_zk.sync("/test_node") + node_zks[0].create("/test_node", b"somedata1") + node_zks[1].sync("/test_node") + node_zks[2].sync("/test_node") - assert node1_zk.exists("/test_node") is not None - assert node2_zk.exists("/test_node") is not None - assert node3_zk.exists("/test_node") is not None + for node_zk in node_zks: + assert node_zk.exists("/test_node") is not None finally: try: - for zk_conn in [node1_zk, node2_zk, node3_zk]: + for zk_conn in node_zks: + if zk_conn is None: + continue zk_conn.stop() zk_conn.close() except: pass + + +def setupSsl(node, filename, password): + if password is None: + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/ssl_conf.yml"), + "/etc/clickhouse-server/config.d/ssl_conf.yml", + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "WithoutPassPhrase", + filename, + ) + return + + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/ssl_conf_password.yml"), + "/etc/clickhouse-server/config.d/ssl_conf.yml", + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "WithoutPassPhrase", + filename, + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "PASSWORD", + password, + ) + + +def stop_all_clickhouse(): + for node in nodes: + node.stop_clickhouse() + + for node in nodes: + node.exec_in_container(["rm", "-rf", "/var/lib/clickhouse/coordination"]) + + +def start_clickhouse(node): + node.start_clickhouse() + + +def start_all_clickhouse(): + p = Pool(3) + waiters = [] + + for node in nodes: + waiters.append(p.apply_async(start_clickhouse, args=(node,))) + + for waiter in waiters: + waiter.wait() + + for node in nodes: + ku.wait_until_connected(cluster, node) + + +def check_valid_configuration(filename, password): + stop_all_clickhouse() + for node in nodes: + setupSsl(node, filename, password) + start_all_clickhouse() + run_test() + + +def check_invalid_configuration(filename, password): + stop_all_clickhouse() + for node in nodes: + setupSsl(node, filename, password) + + nodes[0].start_clickhouse(expected_to_fail=True) + nodes[0].wait_for_log_line( + "OpenSSLException: EVPKey::loadKey.*error:0480006C:PEM routines::no start line", + ) + + +def test_secure_raft_works(started_cluster): + check_valid_configuration("WithoutPassPhrase", None) + + +def test_secure_raft_works_with_password(started_cluster): + check_valid_configuration("WithoutPassPhrase", "unusedpassword") + check_invalid_configuration("WithPassPhrase", "wrongpassword") + check_invalid_configuration("WithPassPhrase", "") + check_valid_configuration("WithPassPhrase", "test") + check_invalid_configuration("WithPassPhrase", None) diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 8d5345082ff..5366de39ea7 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -393,6 +393,7 @@ def test_table_functions(): f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, connection_string = '{azure_conn_string}', container = 'cont', blob_path = 'test_simple_16.csv', format = 'CSV')", f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, storage_account_url = '{azure_storage_account_url}', container = 'cont', blob_path = 'test_simple_17.csv', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"iceberg('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')", + f"gcs('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')", ] def make_test_case(i): diff --git a/tests/integration/test_max_authentication_methods_per_user/__init__.py b/tests/integration/test_max_authentication_methods_per_user/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml b/tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml new file mode 100644 index 00000000000..c6d4b0077be --- /dev/null +++ b/tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml @@ -0,0 +1,3 @@ + + 2 + \ No newline at end of file diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py new file mode 100644 index 00000000000..0142c7db746 --- /dev/null +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -0,0 +1,126 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +limited_node = cluster.add_instance( + "limited_node", + main_configs=["configs/max_auth_limited.xml"], +) + +default_node = cluster.add_instance( + "default_node", +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + + +def test_create(started_cluster): + + assert expected_error in limited_node.query_and_get_error( + "CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2', BY '3'" + ) + + assert expected_error not in limited_node.query_and_get_answer_with_error( + "CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2'" + ) + + limited_node.query("DROP USER u_max_authentication_methods") + + +def test_alter(started_cluster): + limited_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") + + assert expected_error in limited_node.query_and_get_error( + "ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2', BY '3'" + ) + + assert expected_error in limited_node.query_and_get_error( + "ALTER USER u_max_authentication_methods IDENTIFIED BY '3', BY '4', BY '5'" + ) + + assert expected_error not in limited_node.query_and_get_answer_with_error( + "ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2'" + ) + + assert expected_error not in limited_node.query_and_get_answer_with_error( + "ALTER USER u_max_authentication_methods IDENTIFIED BY '2', BY '3'" + ) + + limited_node.query("DROP USER u_max_authentication_methods") + + +def get_query_with_multiple_identified_with( + operation, username, identified_with_count, add_operation="" +): + identified_clauses = ", ".join([f"BY '1'" for _ in range(identified_with_count)]) + query = ( + f"{operation} USER {username} {add_operation} IDENTIFIED {identified_clauses}" + ) + return query + + +def test_create_default_setting(started_cluster): + expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + + query_exceeds = get_query_with_multiple_identified_with( + "CREATE", "u_max_authentication_methods", 101 + ) + + assert expected_error in default_node.query_and_get_error(query_exceeds) + + query_not_exceeds = get_query_with_multiple_identified_with( + "CREATE", "u_max_authentication_methods", 100 + ) + + assert expected_error not in default_node.query_and_get_answer_with_error( + query_not_exceeds + ) + + default_node.query("DROP USER u_max_authentication_methods") + + +def test_alter_default_setting(started_cluster): + default_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") + + query_add_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 100, "ADD" + ) + + assert expected_error in default_node.query_and_get_error(query_add_exceeds) + + query_replace_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 101 + ) + + assert expected_error in default_node.query_and_get_error(query_replace_exceeds) + + query_add_not_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 99, "ADD" + ) + + assert expected_error not in default_node.query_and_get_answer_with_error( + query_add_not_exceeds + ) + + query_replace_not_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 100 + ) + + assert expected_error not in default_node.query_and_get_answer_with_error( + query_replace_not_exceeds + ) + + default_node.query("DROP USER u_max_authentication_methods") diff --git a/tests/integration/test_parallel_replicas_snapshot_from_initiator/__init__.py b/tests/integration/test_parallel_replicas_snapshot_from_initiator/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_parallel_replicas_snapshot_from_initiator/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_snapshot_from_initiator/configs/remote_servers.xml new file mode 100644 index 00000000000..734acf5f363 --- /dev/null +++ b/tests/integration/test_parallel_replicas_snapshot_from_initiator/configs/remote_servers.xml @@ -0,0 +1,33 @@ + + + + + false + + node0 + 9000 + + + node1 + 9000 + + + node2 + 9000 + + + node3 + 9000 + + + node4 + 9000 + + + node5 + 9000 + + + + + diff --git a/tests/integration/test_parallel_replicas_snapshot_from_initiator/test.py b/tests/integration/test_parallel_replicas_snapshot_from_initiator/test.py new file mode 100644 index 00000000000..a7e7e99455b --- /dev/null +++ b/tests/integration/test_parallel_replicas_snapshot_from_initiator/test.py @@ -0,0 +1,73 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +nodes = [ + cluster.add_instance( + f"node{num}", main_configs=["configs/remote_servers.xml"], with_zookeeper=True + ) + for num in range(6) +] + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _create_tables(table_name): + for idx, node in enumerate(nodes): + node.query( + f"DROP TABLE IF EXISTS {table_name}", + settings={"database_atomic_wait_for_drop_and_detach_synchronously": True}, + ) + + node.query( + f""" + CREATE TABLE {table_name} (value Int64) + Engine=ReplicatedMergeTree('/test_parallel_replicas/shard/{table_name}', '{idx}') + ORDER BY () + """ + ) + + nodes[0].query( + f"INSERT INTO {table_name} SELECT * FROM numbers(1000)", + settings={"insert_deduplicate": 0}, + ) + nodes[0].query(f"SYSTEM SYNC REPLICA ON CLUSTER 'parallel_replicas' {table_name}") + + for idx, node in enumerate(nodes): + node.query("SYSTEM STOP REPLICATED SENDS") + # the same data on all nodes except for a single value + node.query( + f"INSERT INTO {table_name} VALUES ({idx})", + settings={"insert_deduplicate": 0}, + ) + + +# check that we use the state of data parts from the initiator node (for some sort of determinism of what is been read). +# currently it is implemented only when we build local plan for the initiator node (we aim to make this behavior default) +def test_initiator_snapshot_is_used_for_reading(start_cluster): + table_name = "t" + _create_tables(table_name) + + for idx, node in enumerate(nodes): + expected = 499500 + idx # sum of all integers 0..999 + idx + assert ( + node.query( + f"SELECT sum(value) FROM {table_name}", + settings={ + "allow_experimental_parallel_reading_from_replicas": 2, + "max_parallel_replicas": 100, + "cluster_for_parallel_replicas": "parallel_replicas", + "parallel_replicas_local_plan": True, + }, + ) + == f"{expected}\n" + ) diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 0a364d7802b..095c663fe97 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -1148,6 +1148,110 @@ def test_dependent_loading(started_cluster): instance.query(f"DROP TABLE {table} SYNC") +def test_partial_table(started_cluster): + table = "test_partial_table" + + pg_manager.create_postgres_table( + table, + "", + f"""CREATE TABLE {table} ( + key integer PRIMARY KEY, + x integer DEFAULT 0, + y integer, + z text DEFAULT 'z'); + """, + ) + pg_manager.execute(f"insert into {table} (key, x, z) values (1,1,'a');") + pg_manager.execute(f"insert into {table} (key, x, z) values (2,2,'b');") + + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + f"materialized_postgresql_tables_list = '{table}(z, key)'", + "materialized_postgresql_backoff_min_ms = 100", + "materialized_postgresql_backoff_max_ms = 100", + ], + ) + check_tables_are_synchronized( + instance, + table, + postgres_database=pg_manager.get_default_database(), + columns=["key", "z"], + ) + + pg_manager.execute(f"insert into {table} (key, x, z) values (3,3,'c');") + pg_manager.execute(f"insert into {table} (key, x, z) values (4,4,'d');") + + check_tables_are_synchronized( + instance, + table, + postgres_database=pg_manager.get_default_database(), + columns=["key", "z"], + ) + + +def test_partial_and_full_table(started_cluster): + table = "test_partial_and_full_table" + + pg_manager.create_postgres_table( + table, + "", + f"""CREATE TABLE {table}1 ( + key integer PRIMARY KEY, + x integer DEFAULT 0, + y integer, + z text DEFAULT 'z'); + """, + ) + pg_manager.execute(f"insert into {table}1 (key, x, y, z) values (1,1,1,'1');") + pg_manager.execute(f"insert into {table}1 (key, x, y, z) values (2,2,2,'2');") + pg_manager.create_postgres_table( + table, + "", + f"""CREATE TABLE {table}2 ( + key integer PRIMARY KEY, + x integer DEFAULT 0, + y integer, + z text DEFAULT 'z'); + """, + ) + pg_manager.execute(f"insert into {table}2 (key, x, y, z) values (3,3,3,'3');") + pg_manager.execute(f"insert into {table}2 (key, x, y, z) values (4,4,4,'4');") + + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + f"materialized_postgresql_tables_list = '{table}1(key, x, z), {table}2'", + "materialized_postgresql_backoff_min_ms = 100", + "materialized_postgresql_backoff_max_ms = 100", + ], + ) + check_tables_are_synchronized( + instance, + f"{table}1", + postgres_database=pg_manager.get_default_database(), + columns=["key", "x", "z"], + ) + check_tables_are_synchronized( + instance, f"{table}2", postgres_database=pg_manager.get_default_database() + ) + + pg_manager.execute(f"insert into {table}1 (key, x, z) values (3,3,'3');") + pg_manager.execute(f"insert into {table}2 (key, x, z) values (5,5,'5');") + + check_tables_are_synchronized( + instance, + f"{table}1", + postgres_database=pg_manager.get_default_database(), + columns=["key", "x", "z"], + ) + check_tables_are_synchronized( + instance, f"{table}2", postgres_database=pg_manager.get_default_database() + ) + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_remove_stale_moving_parts/__init__.py b/tests/integration/test_remove_stale_moving_parts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_remove_stale_moving_parts/config.xml b/tests/integration/test_remove_stale_moving_parts/config.xml new file mode 100644 index 00000000000..968e07fae51 --- /dev/null +++ b/tests/integration/test_remove_stale_moving_parts/config.xml @@ -0,0 +1,46 @@ + + + + + + ch1 + 9000 + + + + + + 01 + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + + + default + False + + + s3 + False + + + 0.0 + + + + + + true + s3 + + true + diff --git a/tests/integration/test_remove_stale_moving_parts/test.py b/tests/integration/test_remove_stale_moving_parts/test.py new file mode 100644 index 00000000000..f7cb4e5817e --- /dev/null +++ b/tests/integration/test_remove_stale_moving_parts/test.py @@ -0,0 +1,117 @@ +from pathlib import Path +import time +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "config.xml", + ], + macros={"replica": "node1"}, + with_zookeeper=True, + with_minio=True, +) + +DATABASE_NAME = "stale_moving_parts" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(node, query): + return node.query(database=DATABASE_NAME, sql=query) + + +# .../disks/s3/store/ +def get_table_path(node, table): + return ( + node.query( + sql=f"SELECT data_paths FROM system.tables WHERE table = '{table}' and database = '{DATABASE_NAME}' LIMIT 1" + ) + .strip('"\n[]') + .split(",")[1] + .strip("'") + ) + + +def exec(node, cmd, path): + return node.exec_in_container( + [ + "bash", + "-c", + f"{cmd} {path}", + ] + ) + + +def wait_part_is_stuck(node, table_moving_path, moving_part): + num_tries = 5 + while q(node, "SELECT part_name FROM system.moves").strip() != moving_part: + if num_tries == 0: + raise Exception("Part has not started to move") + num_tries -= 1 + time.sleep(1) + num_tries = 5 + while exec(node, "ls", table_moving_path).strip() != moving_part: + if num_tries == 0: + raise Exception("Part is not stuck in the moving directory") + num_tries -= 1 + time.sleep(1) + + +def wait_zookeeper_node_to_start(zk_nodes, timeout=60): + start = time.time() + while time.time() - start < timeout: + try: + for instance in zk_nodes: + conn = cluster.get_kazoo_client(instance) + conn.get_children("/") + print("All instances of ZooKeeper started") + return + except Exception as ex: + print(("Can't connect to ZooKeeper " + str(ex))) + time.sleep(0.5) + + +def test_remove_stale_moving_parts_without_zookeeper(started_cluster): + ch1.query(f"CREATE DATABASE IF NOT EXISTS {DATABASE_NAME}") + + q( + ch1, + "CREATE TABLE test_remove ON CLUSTER cluster ( id UInt32 ) ENGINE ReplicatedMergeTree() ORDER BY id;", + ) + + table_moving_path = Path(get_table_path(ch1, "test_remove")) / "moving" + + q(ch1, "SYSTEM ENABLE FAILPOINT stop_moving_part_before_swap_with_active") + q(ch1, "INSERT INTO test_remove SELECT number FROM numbers(100);") + moving_part = "all_0_0_0" + move_response = ch1.get_query_request( + sql=f"ALTER TABLE test_remove MOVE PART '{moving_part}' TO DISK 's3'", + database=DATABASE_NAME, + ) + + wait_part_is_stuck(ch1, table_moving_path, moving_part) + + cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) + # Stop moves in case table is not read-only yet + q(ch1, "SYSTEM STOP MOVES") + q(ch1, "SYSTEM DISABLE FAILPOINT stop_moving_part_before_swap_with_active") + + assert "Cancelled moving parts" in move_response.get_error() + assert exec(ch1, "ls", table_moving_path).strip() == "" + + cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) + wait_zookeeper_node_to_start(["zoo1", "zoo2", "zoo3"]) + q(ch1, "SYSTEM START MOVES") + + q(ch1, f"DROP TABLE test_remove") diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 60a6e099b22..533eb601ad6 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1549,3 +1549,19 @@ def test_all_groups_cluster(started_cluster): assert "bad_settings_node\ndummy_node\n" == bad_settings_node.query( "select host_name from system.clusters where name='all_groups.db_cluster' order by host_name" ) + + +def test_detach_attach_table(started_cluster): + main_node.query("DROP DATABASE IF EXISTS detach_attach_db SYNC") + main_node.query( + "CREATE DATABASE detach_attach_db ENGINE = Replicated('/clickhouse/databases/detach_attach_db');" + ) + main_node.query( + "CREATE TABLE detach_attach_db.detach_attach_table (k UInt64) ENGINE=ReplicatedMergeTree ORDER BY k;" + ) + main_node.query("INSERT INTO detach_attach_db.detach_attach_table VALUES (1);") + main_node.query("DETACH TABLE detach_attach_db.detach_attach_table PERMANENTLY;") + main_node.query("ATTACH TABLE detach_attach_db.detach_attach_table;") + assert ( + main_node.query("SELECT * FROM detach_attach_db.detach_attach_table;") == "1\n" + ) diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py index cf189f2a6ed..a975db05020 100644 --- a/tests/integration/test_restore_external_engines/test.py +++ b/tests/integration/test_restore_external_engines/test.py @@ -70,6 +70,12 @@ def get_mysql_conn(cluster): def fill_tables(cluster, dbname): fill_nodes(nodes, dbname) + node1.query( + f"""CREATE TABLE {dbname}.example_s3_engine_table (name String, value UInt32) +ENGINE = S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'gzip') +SETTINGS input_format_with_names_use_header = 0""" + ) + conn = get_mysql_conn(cluster) with conn.cursor() as cursor: @@ -136,6 +142,7 @@ def test_restore_table(start_cluster): node2.query(f"BACKUP DATABASE replicated TO {backup_name}") + node2.query("DROP TABLE replicated.example_s3_engine_table") node2.query("DROP TABLE replicated.mysql_schema_inference_engine") node2.query("DROP TABLE replicated.mysql_schema_inference_function") @@ -149,6 +156,13 @@ def test_restore_table(start_cluster): ) node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated") + assert ( + node1.query( + "SELECT engine FROM system.tables where database = 'replicated' and name = 'example_s3_engine_table'" + ) + == "S3\n" + ) + assert ( node1.query( "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine" @@ -175,6 +189,7 @@ def test_restore_table_null(start_cluster): node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}") + node2.query("DROP TABLE replicated2.example_s3_engine_table") node2.query("DROP TABLE replicated2.mysql_schema_inference_engine") node2.query("DROP TABLE replicated2.mysql_schema_inference_function") @@ -188,6 +203,13 @@ def test_restore_table_null(start_cluster): ) node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2") + assert ( + node1.query( + "SELECT engine FROM system.tables where database = 'replicated2' and name = 'example_s3_engine_table'" + ) + == "Null\n" + ) + assert ( node1.query( "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine" diff --git a/tests/integration/test_s3_cluster/configs/named_collections.xml b/tests/integration/test_s3_cluster/configs/named_collections.xml index 64d1bd98df2..2d3a69a8c38 100644 --- a/tests/integration/test_s3_cluster/configs/named_collections.xml +++ b/tests/integration/test_s3_cluster/configs/named_collections.xml @@ -6,5 +6,12 @@ minio123 CSV> + + http://minio1:9001/root/data/data{1,2,3} + minio + minio123 + JSONEachRow> + id UInt32, date Date DEFAULT 18262 + diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 03919ee6a4d..c31851fdfe9 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -459,3 +459,51 @@ def test_cluster_format_detection(started_cluster): ) assert result == expected_result + + +def test_cluster_default_expression(started_cluster): + node = started_cluster.instances["s0_0_0"] + + node.query( + "insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1" + ) + node.query( + "insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1" + ) + node.query( + "insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1" + ) + + expected_result = node.query( + "SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id" + ) + + assert result == expected_result diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 4800ab798bf..7bb059910dd 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -3,7 +3,9 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance("instance") + +# `randomize_settings` is set tot `False` to maake result of `SHOW CREATE SETTINGS PROFILE` consistent +instance = cluster.add_instance("instance", randomize_settings=False) def system_settings_profile(profile_name): @@ -128,7 +130,7 @@ def test_smoke(): instance.query("ALTER USER robin SETTINGS PROFILE xyz") assert ( instance.query("SHOW CREATE USER robin") - == "CREATE USER robin SETTINGS PROFILE `xyz`\n" + == "CREATE USER robin IDENTIFIED WITH no_password SETTINGS PROFILE `xyz`\n" ) assert ( instance.query( @@ -152,7 +154,10 @@ def test_smoke(): ] instance.query("ALTER USER robin SETTINGS NONE") - assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" + assert ( + instance.query("SHOW CREATE USER robin") + == "CREATE USER robin IDENTIFIED WITH no_password\n" + ) assert ( instance.query( "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", diff --git a/tests/integration/test_settings_randomization/__init__.py b/tests/integration/test_settings_randomization/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_settings_randomization/config/users.xml b/tests/integration/test_settings_randomization/config/users.xml new file mode 100644 index 00000000000..a7e5d5f15f0 --- /dev/null +++ b/tests/integration/test_settings_randomization/config/users.xml @@ -0,0 +1,7 @@ + + + + 59999 + + + diff --git a/tests/integration/test_settings_randomization/test.py b/tests/integration/test_settings_randomization/test.py new file mode 100644 index 00000000000..f93a0a15984 --- /dev/null +++ b/tests/integration/test_settings_randomization/test.py @@ -0,0 +1,35 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node1", user_configs=["config/users.xml"]) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_settings_randomization(started_cluster): + """ + See tests/integration/helpers/random_settings.py + """ + + def q(field, name): + return int( + node.query( + f"SELECT {field} FROM system.settings WHERE name = '{name}'" + ).strip() + ) + + # setting set in test config is not overriden + assert q("value", "max_block_size") == 59999 + + # some setting is randomized + assert q("changed", "max_joined_block_size_rows") == 1 + assert 8000 <= q("value", "max_joined_block_size_rows") <= 100000 diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem new file mode 100644 index 00000000000..b17baa62262 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHEwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwOTAy +MTYwODI0WhcNMzQwODMxMTYwODI0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw +DgYDVQQDDAdjbGllbnQ1MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +zivZ5IrYyoJeDX0Zbl/cl8rKE0LbmtD+QKZvQXHD+gstXWxPOEFZVxO3BuqmvBZ1 +MaYNyPYA9JyyP+pO9cE8RrTk3w9xMhv8dYWJQK7na9W9RTSXs8xhPwiEm4XuVgqv +GfK/EvdxbFMhgzExOR333TkmXPxrMm5xPWwV3RsTCjNVW7nmdPHXmchuTO7lQtww +6fETqc1Zqv8TO2x/uGZiwAzNYbueWHyzL4Y1UJ7D0mnNNgJvvxtcrzBTlGLLhJ5o +0+zVQLaOnac0WVk0wDhVaxd/gR4bYN3ixvuFbbOaTroFVTMVme196G2FkJI/05Pn +D68r1yUpvuNDjkBbuBO43PlsFKLhPU5twfu4TesEb2WZ0VsNxn8Hc0Ne02WwHsHa +Fi1N0oXvVIeGvvzevuTkjPbh2aCaQX9qbhLXjhgXNFAXQq+qI8ECCWC4LruPL1Es ++ZM2thQAg4k8GY4U9Q8zX55Ut7t9T771QbHFOAupuTgZJSz0jC8JId0m7NOOqtvu +4w/GqefCt9bp+wlQDXVOgi+S3GqKM1OuIbs5XQJtNfi6l684ptAFHSHMHXvfGLHI +MlKdobjNqEFVK9qO/3YvnTUyRyaUA61aHxraXWBwvcUByXwXdgbuz/M5XAi6fr0V +Trw9iUSviDvu4HwDo8PCw9i8xYFtkr2k1kurWEsFZMcCAwEAAaNrMGkwJwYDVR0R +BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYm9vL2ZhcjAdBgNVHQ4EFgQUENIS +25yehLQrlL8vC+DpkNE7urowHwYDVR0jBBgwFoAUiSo9XUmDdI1vjLtMRUK6G2RZ +kKUwDQYJKoZIhvcNAQELBQADggIBAADH/LINEwB1WUQ2Q/aHiNv1ZyJ+ifgs9ewp +/t0uwS+53ctcmJ6Sqeni3/UIdLYjpcdBbl1XpqTYuGF28C2YZrWKFB0HaOiH6D8B +zcGGAkIKFqnhcJxyl37je+scZ8Wk9b04L+X+miN6cgIWm6rQezDF3rs1xvAVBqTM +YPIk6sBIgHNJy4D3S5KdhqNV0/8KY6T65nGFdEq064qOk8HvS6DyYOs22AitCD+L +gcWGJHJ3BfNASbRrT25zb1HLUIFFbFIGaPFd9GbiU5hGb9MgUzX44q+WdXoEa59a +6y9ZcidjEqAGP/FMz16D831YpqRBherZ09ztWXeTfv4NxauisLuoqpOr7CmpQ+Ct +O5t0cUHILeNBFR7rdMOmDawpEcOSGqcJHdPH4SjP/LtgQODWiNys19Yp5afbM5Lz +IjLjq1wAHVtSvPHjRhnZSq0SiU1XlDmu1Em3HbFe5RmqL/lcLe7/U10ddngADG7E +XgPE0jcvl7rYASqYuTbKd6Q53QYx0K7xc1n8mIRJuAofPwl6Yns/ytvw0+E9TBS1 +oGb7j6V/k+Xd77dfJ6fckJXPg7Fm3GPO1ax7FNU51sCrvAHsMZhiWQa6pZzBEORM +4yI+DSFyskyWXCPth9r3UqHQXzX86LRkyDWg9l6v3NWRSI1j/e7dZds/U/sg2maq +css4A+kM +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf b/tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf new file mode 100644 index 00000000000..8cb20e70290 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf @@ -0,0 +1 @@ +subjectAltName=URI:spiffe://bar.com/foo/boo/far diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem new file mode 100644 index 00000000000..aa65de6e26c --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDOK9nkitjKgl4N +fRluX9yXysoTQtua0P5Apm9BccP6Cy1dbE84QVlXE7cG6qa8FnUxpg3I9gD0nLI/ +6k71wTxGtOTfD3EyG/x1hYlArudr1b1FNJezzGE/CISbhe5WCq8Z8r8S93FsUyGD +MTE5HffdOSZc/GsybnE9bBXdGxMKM1VbueZ08deZyG5M7uVC3DDp8ROpzVmq/xM7 +bH+4ZmLADM1hu55YfLMvhjVQnsPSac02Am+/G1yvMFOUYsuEnmjT7NVAto6dpzRZ +WTTAOFVrF3+BHhtg3eLG+4Vts5pOugVVMxWZ7X3obYWQkj/Tk+cPryvXJSm+40OO +QFu4E7jc+WwUouE9Tm3B+7hN6wRvZZnRWw3GfwdzQ17TZbAewdoWLU3She9Uh4a+ +/N6+5OSM9uHZoJpBf2puEteOGBc0UBdCr6ojwQIJYLguu48vUSz5kza2FACDiTwZ +jhT1DzNfnlS3u31PvvVBscU4C6m5OBklLPSMLwkh3Sbs046q2+7jD8ap58K31un7 +CVANdU6CL5LcaoozU64huzldAm01+LqXrzim0AUdIcwde98YscgyUp2huM2oQVUr +2o7/di+dNTJHJpQDrVofGtpdYHC9xQHJfBd2Bu7P8zlcCLp+vRVOvD2JRK+IO+7g +fAOjw8LD2LzFgW2SvaTWS6tYSwVkxwIDAQABAoICAAxYkuU9LUs/MEulfKLf6bFL +SvuJSuZD8p3DebEVE3PDPJW2GS5Yt3Vf8FthLNfhjmYJOBUbKiZ7xa7PJKhRyH7B +cV0sKw6hn2YYN4ZgkuohtAJTamwxZndkVrCLfiC35hFrOGb121xtw/l7tiNh+IhO +Vk5GLVMpSu2vLcX+S48WUNu7z6GcI9mJJ5XCzobI8cLolVFbQy3atXefASz1MJ+o +hJoAJrTMztwfMM0hnL9aZ5f+4Fs008GH5tFhcyd/Zu6d5Y1JAVROgXOCRIboOLE/ +iHqeZ2xPDTf2MuDFWw2w1QXrl7UOhDYpbNCh+WF9n14QktMA3jWYnWCE2Rk2USEK +2QhsScNgA7dY3S3DbSK1ZiyZEgbvdq6EYy5I46jNXqRrfKfuD7J04dtO1pmBWZy3 +a765FMiejbHrC1xrKtSnvGj1oJ23P3TRiEL6tx3QF4bAulWE62ULBfDgqr56pPbt +KOfPinEFa/14+B4a21iFzsoA6SWPt+i/k00RyRHw4X7acTd8lsavHcT8PGY4wMLX +cNEombZn4h/0oCqAmBtoXYwctcyQwzhJNzAp1GFMG2gdBPhXUsWcwwjNNzeaewCq +BO0YO/DFgj1qTHZwswTfppJMF+BTE/sol4eaqOV1TYV48OfYTdo+6dqfH2JkaJyy +PVJAoZtKKPfDRlrHMrE9AoIBAQD1oZMIi4Nm4TpdOGsU5O2keDW6DXdoxpOI6u9H +YqWZlSZ57q2u+8JkoVJhH/jq4l1QuXtUObpQ9DYOpPwPkt+ZblUut/ZrmMo+gs81 +RtIv4EGmgFmtrXgmkYGsoNrIu4/ayBPDwGz7Z+gEta+gxhYbxSZgb2ZOZzH/c2my +3CvfgFiYyv/cfVvynjprRZoxowQvJoMCEEA9REZdO5T51lshQFcrpFU2ucQNKV60 +DELV6uJzhL0EDvg9IqP1CxZhdCsTifE/SNVepNWXFegFGVsD/vxmXVxJYevSDQZY +SvGWVcgyuQA8Gdze6y3jOUlzCQ3v7D63PgAPS+yrGXLd0Vz9AoIBAQDW39oRb1n2 +ves41cEz7qPNiGmc+zhzyZE30HFnl/RxREPfhzvifTUk3YTUPdJGpcgf+sMoPQ/R +plx/ZrFPTyV7U+svf/eoPlpNnVnZ1m+C/nN7AjVwq04GRHKYD3lTD7KpoYQ0ICGO +z9xFiGbK4VRNIiNpyGc3IZBmrwB2wsGn1L4yfFZly3Th4qiDmNpR29rgteQYWSNJ +hSqsCFcqogJfppP+QKRGf00uHDa1AGQhB2eFykyAniQw52FcdpUdkGqYiR1vdir7 +5XckNixq7yyRflo8CftHybyHipTBZVXBzdvDpY5mxANPcJlSPhJxqruxK9FzNYv6 +OiDlFnFPwNYTAoIBAQDLByRbWcXhEaWGTA3mlVsAKQRvppXemtRzxYzGOnmaure0 +7V3OVvZU6ysGcMD0lOqAwO95rMWeCF1uOVzHGqWLUrFCDni2cvIoPUM8TC+LtxdM +oqZ9cfbwGjFPGR398Vp0ghAkKzdpjncu/WYPw+ueRJT1/N5ZS979wM3LM0VoVdPl +Z1WZUFzh63tqE3viL1ZUCscau4f9nCN1CLyYzV9B2sayogB1XZL5Ngq7U68i1iUb +SspZQm5ZSfqvntx+7OB2I/yuTGtg8WpCma7QGCcs5GSHz/9qAHBFjNKDxF5v3rO9 +iUDybIYuE8I4IK/fT8qfV2x3Vd4CxsT2n/Bb/KOZAoIBAQCyIwrqgWIYCQNFLxDp +OOJbtse3+R2oAGd/2jLqKzPf4zRR0A95iUvRsEQ6oLX/E9mBiabZaSYALOdqR3yf +v4HXaI8F5hUvSeIbjmO7EOeJteGPDRm4uowI7h4CqnFuxwjbUKgFfs3TU8fNbXOq +pnv5JmAthpLrRcwtFNBRpMxfkyPLPwFxiubvjbUexE3ap2Yh/SmIdf2nKdtim9eH +5KALJFJ06qpGN6uImqNQE27vYvAUHs6lonVmhaxVt4mP5PY6VxIsIc8o3eeUAcV5 +MafFRvcP50aHEVxXEjCY1KXv8fZLkKkp0T7dUQALCqLH0T+hdi5CURYm8KHsylpO +QBQ5AoIBADXNlx5bXEU2lW21wyWG4v3gqvmnDgQG66RXaHC0Rgpn6BW7pose40fv +I82yWviabk7wlvaQgXllzZ5vGL43/8pp+wFpRUl5nX8P1ZA8pRkRArm1vLxQRjkM +90j0M/XhTnC8eC4qKsQx8dPUH0SPkHJAd2tWcP9Q3635zETcINkpsNbsB/QNGBZ1 +JefHA/GNZZBrjFMOnvzEbEbqBdhwlZNRlZxVq/svvNzzv12jFQpM7HVy95bATiZ8 +x7SQlDxCWOWApd9VpM83JnPWCfu0Yi/HHPJd6YdueMhRSvUpLs1sD8fs9bFQ7+OL +erqwgB4b3yDlXijY75pPDxdPf/0qA3Q= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem new file mode 100644 index 00000000000..6ad15243bac --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50NTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAM4r2eSK2MqCXg19 +GW5f3JfKyhNC25rQ/kCmb0Fxw/oLLV1sTzhBWVcTtwbqprwWdTGmDcj2APScsj/q +TvXBPEa05N8PcTIb/HWFiUCu52vVvUU0l7PMYT8IhJuF7lYKrxnyvxL3cWxTIYMx +MTkd9905Jlz8azJucT1sFd0bEwozVVu55nTx15nIbkzu5ULcMOnxE6nNWar/Ezts +f7hmYsAMzWG7nlh8sy+GNVCew9JpzTYCb78bXK8wU5Riy4SeaNPs1UC2jp2nNFlZ +NMA4VWsXf4EeG2Dd4sb7hW2zmk66BVUzFZntfehthZCSP9OT5w+vK9clKb7jQ45A +W7gTuNz5bBSi4T1ObcH7uE3rBG9lmdFbDcZ/B3NDXtNlsB7B2hYtTdKF71SHhr78 +3r7k5Iz24dmgmkF/am4S144YFzRQF0KvqiPBAglguC67jy9RLPmTNrYUAIOJPBmO +FPUPM1+eVLe7fU++9UGxxTgLqbk4GSUs9IwvCSHdJuzTjqrb7uMPxqnnwrfW6fsJ +UA11ToIvktxqijNTriG7OV0CbTX4upevOKbQBR0hzB173xixyDJSnaG4zahBVSva +jv92L501MkcmlAOtWh8a2l1gcL3FAcl8F3YG7s/zOVwIun69FU68PYlEr4g77uB8 +A6PDwsPYvMWBbZK9pNZLq1hLBWTHAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +SGJaGaSkfsUBSqPbkbtyE2ndHIY8QMPvJAMB7pPbA/MlX39oO2U1AraFatN/Xzpd +0uQA2m0UsXF2+ScrG4rd14YkU4wpcT2lIplvFFzUkwfYLATRYrFV0FCru1n28+Ve +TyUG9/tlf/jnNZwWtB5AEbCWmzXSjXUuwXdn2oQ0z+3dxdSnpOrDkzyYeocIMnMJ +pG0qdBd+hJrK6snJNli6EfsmmykyCSLorFpnkm2uKUwdTAD2/MlxYdutjk7tqKw5 +wpYi6Wqt/euyQ94Ri4aICZMBFk5+zDdaEOKVQZ0aUT8RWt/aD2ksbTrclDICOe84 +iG+Nf/CyNqSNqsUugCSCxN3UUZgLKEyJb8Pz4N/nFFRBb1aiZ/5YVRTYtZ+k4tLb +s2exxqVpDg3M5G5bk0iJ8DBEjO/yKwobc7HAtnTAEIs0HGord3yzg3F0Y+5ecQAg +9ESOptz33EBkTHxpBmt0D0ACa4CTghrf/Id8imNtdGU9i7MeEba+iUVAUP8VfhtL +FJPRR8aVaKaLc9uCAiiHuRc+J0EHAwTOKKTK3Z1mkYO00kMCiRcru8/H6ibkrkV7 +kRL6NvAc7CsEzSDPkFKAZhQ4p6AcfNC5yRiNLG9JB/wQgBg8v23Uwtk74gOXIaQN +WUjwzdYOljdcFOzmoDMzyfIFwSXcO3dmmjqOwh2HNw0= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem new file mode 100644 index 00000000000..b83c380a3bc --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHIwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwOTAy +MTYwODM0WhcNMzQwODMxMTYwODM0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw +DgYDVQQDDAdjbGllbnQ2MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +uz+3VyFhBD6K68mkDM0LJfRfGbuD8tSmHhtO6+XQzhIUiANW+A1WdyD+zWky0QsS +vl/2Ds1PmjU7659pkSD8Sidjdz3/TP0eO49nHinLjRQa2Oawk6PCjLIcpJ+A5eGb +Hno/oiQBMCAPpVh3sex+5yPiUQN62cKRWryv2JJqJmEgvpjC92SaIjf9M3mCsxMf +an0CZj6hNcorxXQNnF4JZRPQ4mMgBqgFS5Oz1YujHiBUN9ZoIGmS3HZ9LYl5lL7o +GxNXluyIrIw1kiyQrx+wJFdYwTBeyCqw8wmf993tHRmvpJ2ZFVXTbYqpj2Qkom+J +EpCRBqCmK+/uSgLOAicS/wR2eiYXkw2rYDIQ6yCyeW7nnaFVWNAFNSdAW1Jdr+Z2 +fKMolhYAcpeMQAYN5wTqJBF9dDwMxbOcdSh1wPZcenuO2q5cMJcn3qt3+SigNKcQ +BvqtZ54GNGq9h24f11+cqco80v4WYxrmXu8bXQfgtX07UVvqkjbG7O4HtbGDvOlw +KO7d1kOj4XUJdZbz9g5kaGufN4hlBs9JzYNOZdywNwBiPxHHE71Ht2ihfbBKSl9x +4Zse5YhPqToacWd5FRW+If5EpHkxxW+f4e61S2/8fnn5yHpMX22HXNBmOUR3vBrl +xfkje22ZuEf5NfB95aEaRZABmXQaHKdAVQeaAT9TvPMCAwEAAaNrMGkwJwYDVR0R +BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYmF6L2ZhcjAdBgNVHQ4EFgQU0ieN +0CTYiMz7HJs9OH2U4imSzN0wHwYDVR0jBBgwFoAUiSo9XUmDdI1vjLtMRUK6G2RZ +kKUwDQYJKoZIhvcNAQELBQADggIBAELD77yWem26DIWG2Mi6q9KvLb/d8bOFN4mg +SYYekdEryyYCFhbb0P9T7GKj4KxNxkZaGgwXl+PM8SH7FhqemRCwCtHMVQiWvC1x +XLCrGiep7Dx+vvKHoCdapQp3uRfsZMHSZbsHJgQRFRt/jgP3jFcrLAa9tD8Yza4y +lBhh8pUROC0edlqom+BkLYA6A7FWA89x/NZL6wiyPKqRh2SuACszrlQdj4oqdgIF +pAILjDy4fLaCGGUgu9kHonV0xTjqSdFXiSeImSkePOvGWrh2B5GqRbanPHDSR5VD +pIhKZPpnvUBNhzoAMv1RFbgNs9m3sYqYChxb8IOGiY/3EG4rgygVez2yu16v5WMU +PIWwSv3N+oceG085dHxhn4TcDamdFrQHssp7GJBbyGEINHQRiZ4cu3Vz2cVNYcKD +iFJZ8vVwU7IZOEQeww6DT+gL+wqSgTyDvEXQNbYupFYPZXSBIaygH4eHa+PqPnNL +DJTpPAlwNRB2+eL3bZxvNAfwqIY6xgwnLBr1QrosmfMKgkswkg8gVoIpIRGACzk2 +iY818Jn+IG/M/aPF3p5dTsOqH3bQmz4ZpoLB2dytqkNDGKSAPPGOpe4MDyx0prCH +GWDjEjn4xT9AjAJVpTWJCENPzFzwA7byApuZwLTy/5ZBbNJf1K/JwsqXUhHKENb2 +NzMKvQCT +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf b/tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf new file mode 100644 index 00000000000..d421f4198b4 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf @@ -0,0 +1 @@ +subjectAltName=URI:spiffe://bar.com/foo/baz/far diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem new file mode 100644 index 00000000000..e56fed5dddf --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC7P7dXIWEEPorr +yaQMzQsl9F8Zu4Py1KYeG07r5dDOEhSIA1b4DVZ3IP7NaTLRCxK+X/YOzU+aNTvr +n2mRIPxKJ2N3Pf9M/R47j2ceKcuNFBrY5rCTo8KMshykn4Dl4Zseej+iJAEwIA+l +WHex7H7nI+JRA3rZwpFavK/YkmomYSC+mML3ZJoiN/0zeYKzEx9qfQJmPqE1yivF +dA2cXgllE9DiYyAGqAVLk7PVi6MeIFQ31mggaZLcdn0tiXmUvugbE1eW7IisjDWS +LJCvH7AkV1jBMF7IKrDzCZ/33e0dGa+knZkVVdNtiqmPZCSib4kSkJEGoKYr7+5K +As4CJxL/BHZ6JheTDatgMhDrILJ5buedoVVY0AU1J0BbUl2v5nZ8oyiWFgByl4xA +Bg3nBOokEX10PAzFs5x1KHXA9lx6e47arlwwlyfeq3f5KKA0pxAG+q1nngY0ar2H +bh/XX5ypyjzS/hZjGuZe7xtdB+C1fTtRW+qSNsbs7ge1sYO86XAo7t3WQ6PhdQl1 +lvP2DmRoa583iGUGz0nNg05l3LA3AGI/EccTvUe3aKF9sEpKX3Hhmx7liE+pOhpx +Z3kVFb4h/kSkeTHFb5/h7rVLb/x+efnIekxfbYdc0GY5RHe8GuXF+SN7bZm4R/k1 +8H3loRpFkAGZdBocp0BVB5oBP1O88wIDAQABAoICACymDcaV6/dCHogIpbhzsAPV +2FNfdiAR+qZVJqVzQC3S+9hOy48MRyDS2k8KlZZpCIgig56V8DQ6G1acxWRYtC/O +YpZNTzIBbRMQp6r2llXGhHxRzar2sm4wDkpmyiqGeCF1TvUPlsTt8C0iAjCHzt64 +nL9qkAGatmQnd9qxVuRd5pvr+xlYgLRGG3cJs1OV7LjMpCTTLEeSNIu5l4FAnwbe +CcHhlwJfUBvsVUZHLJcDaHGEC4InCmDNVB3mmPoR53CFVS5kqlZSfHelbr6DVNHl +jgFK0l7yZw0cr2tAHMkClfIvrg/7ThXhqXrKiz28ULf/hsVIyzbQ2EYHky3KOWny +04O7/NnOkVHs+XUyNC4sv9nkcd9ntKkvvUPPK0U6vbW7IasC3jCh5LMyZjHYwgmK +hzxEBZSyutKWn3RWncarwQ/1Vbq3HjbkeVTipXAa7Bny17wiAeZMZ2GqQZ9VcNQ3 +YJWDgxS5shwcEo+71sC4o2HjmWKcPujmt84XcWc6yphPbCpbwwswaQD5MaZxeDgZ +OUhF9LfslzNrwwoZTFz/Qhy3TOshF7BIbUdQnWLMNdAb9ccby/0WgOmfD6V4t99N +ksb2nWgvvK3isycs6HHVP/fgv+yM9cKGs66JoH2Jm+SInUtpR5Gv1aGeV97/9WFd +JuiHtHQIty+8n6GDTscJAoIBAQDuHCBeZ+pVTyW6wxjd4OD2keuDjM3Z7X/UgCoJ +kR87Dwjd8SHEw8QaH8vvGYBNktFu3KbQ1TV2OR8gAdnwlHeI5V2/nIVX0UBjQM9X +GC3cmzsMOBAem0nuYXZG9yvawwPUdZ18fQc2wAs4GqL4uKaOeuCefNyK5wKfvX7M +sA49D45gvLUhpfkTeM8HK9UQwMfMg2fFBzZifqTIG4OGkkAeEY+rkJTUxnvTuuFU +dkXXF8Qe+pSPkbQVQYYRRO9Wk0i16R6VaYrl3vvi72w2gEw7iQya0A1bHZe3s7vv +jQuz8h954kcgLYCqsOm/mj3t654jrjW1Z5yRjznTUJKrKMh3AoIBAQDJUVCp2Frm +NgzrZXD1QrkJ1qCRBHyVu7FikXqNszc9lLD5y8YWRnhDtGruOQ3DYjpuD/DMrO2P ++iBTambM3mJt6FE8TkXHyMzLoJ/I8SMLMbLNdDpsj8D8RlftwIESiNu9DQfMle5l +8jxZ7R7usio8HysVm5u6fsSmYVUZF+sWLLAUqote4HQxdvDup9A1q7onVZUYfKnK +mCVKqfdqFDqMKTSHOngxA5wzQWdMqdgLxiYKPYbkNsJ3dhXQwJjfbyDQq4X/foec +0wG91/WqsLiMOZLsQBiGMgOq85IqGBByl51QnkT8drPEJsXX6UCHjQ7AYHe0U+pe +JTa6nMfk2AplAoIBAQDemJa+CuFrvSEldowBqOBGQeXtRi2PBNNTAjnKVcvfd0+v +xGPwQZ9LWNxaevmSC6YUwNyGMDvZeM8bKf/nQ7R32w0IOzgA/9L0trrezfDZ4piR +9LtFEaFM4/ohn6J00Yj8DrQak/uxeFlEqsoeQOkcG81u/IVkqU+vrAQlMJUvCiLt +VpzyhunSBUdtidGW5tIh49qXvAVgkMpVdDtCC+k68unf1rr8K03Jg1RxlFv4F/S1 +jUZi7TBwCqBd9pbU1b3MqdF4loFOImZSIceFL+2UXqvU8pj5zDFwf+s6qB3/rGM2 +m44oi8FUUS1EfNpWWMWuz4bQPruE7GN/pDxpHChDAoIBAGZF5yLCBTvazalwuXf/ +051J6fyCOQCgf7dBjO8b0r54IYhlm1aJqmNK7t/jIDjYWDK96mkwmOeB/JYkAeGm +QH7xfQOUCCM8wb3Y9hPRXSo8r0ds+plYVoRTACyKlYfi+y8JxaKLsLcd3scYjZRZ +8tbkRrENgom2CRU1cVP0MLvtK+7zzSYABUdz02aK3l3FxiZhdgMgoemIbbmGq2i6 +qhu2ezcP3yuXV+06Fs59ooIowf6Fz1d08kpaNS7+CSvJevRHjyWDFEX5dHMLmYSD +jt2+CgP3c/4IvpBEeUblPsXYfFUOcqGHdD/8KppLpzq10H6vA2EbGH+HjEw1hd+e +WXkCggEBAIq+MHXpbipKp7lsxsp/moeNXUojLfiBcQLU+zauWBdP3kTJmDd7D3Vy +MoN4zpvhboue70RP1ZZG3yp8s9yR62XrO+44DrKKIhVitpa1IPSavFovFVl/vx1H +F6PuZvACBF7mRZa23h9XSizvdfycEDf3rZidLG/Y9IarrLF3HYjClBktJTtm0YaU +QnmTSUdNvRBCs6/BGQNXY76VB5AHNjeLjjuLLbxYF5XsHlsdacgdMv2ShmIcibrT +nSMK3RdRdRt1Nu1k6v7MkGcaSky7zJIeu6+VftA/7bVKBcnvb+iGZSUSk2QRTCGW +nT+c65hmPp61jcBOgCF3CntcIw4eEDc= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem new file mode 100644 index 00000000000..1723c8c7273 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50NjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALs/t1chYQQ+iuvJ +pAzNCyX0Xxm7g/LUph4bTuvl0M4SFIgDVvgNVncg/s1pMtELEr5f9g7NT5o1O+uf +aZEg/EonY3c9/0z9HjuPZx4py40UGtjmsJOjwoyyHKSfgOXhmx56P6IkATAgD6VY +d7Hsfucj4lEDetnCkVq8r9iSaiZhIL6YwvdkmiI3/TN5grMTH2p9AmY+oTXKK8V0 +DZxeCWUT0OJjIAaoBUuTs9WLox4gVDfWaCBpktx2fS2JeZS+6BsTV5bsiKyMNZIs +kK8fsCRXWMEwXsgqsPMJn/fd7R0Zr6SdmRVV022KqY9kJKJviRKQkQagpivv7koC +zgInEv8EdnomF5MNq2AyEOsgsnlu552hVVjQBTUnQFtSXa/mdnyjKJYWAHKXjEAG +DecE6iQRfXQ8DMWznHUodcD2XHp7jtquXDCXJ96rd/kooDSnEAb6rWeeBjRqvYdu +H9dfnKnKPNL+FmMa5l7vG10H4LV9O1Fb6pI2xuzuB7Wxg7zpcCju3dZDo+F1CXWW +8/YOZGhrnzeIZQbPSc2DTmXcsDcAYj8RxxO9R7dooX2wSkpfceGbHuWIT6k6GnFn +eRUVviH+RKR5McVvn+HutUtv/H55+ch6TF9th1zQZjlEd7wa5cX5I3ttmbhH+TXw +feWhGkWQAZl0GhynQFUHmgE/U7zzAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +Gewd4gSxvJJ1LIKLVTLeMdEdJfzsS52Kh3BCerku/C4ZKcIyT49dTbi6l6d40bHJ +Cs32Hwps8/qufhwwWn0r/wyww1Mgfr6ccMgKmrz1VbgWmD9owDzlL014ygiDk8yi +LwfSLe43NFrFC/FcAJUd/P8UEe0/8GcHjzhU/zqh3VAL7RzSL3k73CsTFiDaxtQL +8qISpA0lYTldMx5RaN9COWi0rPFv7mJAYRXtE/Cb+T2hb53khOiiRrJYIEJjjAhj +g8p9FOzlbXdpfoChVk7NA90CbCbxrQ2BiUqQAVYnGhntzHMwR9YxOYjvjnuiHuHX +7+4Vheda88TciMJlj0TC2e1mXBo182n/qfETeI26MsEOs3DidLT+ygM3woFQyIrX +3x2kDlvmILKg1mPGhqaRwwzCmk5a1TVdDTRo9VkOvR5/tbfG3vHlgpvBtUFCkEjX +HOyRh0A3TquofUbtx638lMWscBLqM5g6VO+Hytk6zBmq+8caJFNTOeTHZur04ZLM +SWfkIwl0B863owNuq4KxXI3NvpCc5LtGc9UrwVoHSH/pv6tbKEX15Y0ERL5/e33M +GT0D00cPnWAzmYQpYzHQ3Dj29XKlIxWBkn1QvrIFyb/T+dld1efZ3HlQxZEQvOsR +McY90r+HmVt8uCioYnC4DmchWlSX1MJe/h72udVbAXk= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh index a09b7b2874e..e8d649fcff1 100755 --- a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -14,12 +14,16 @@ openssl req -newkey rsa:4096 -nodes -batch -keyout client1-key.pem -out client1- openssl req -newkey rsa:4096 -nodes -batch -keyout client2-key.pem -out client2-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client2" openssl req -newkey rsa:4096 -nodes -batch -keyout client3-key.pem -out client3-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client3" openssl req -newkey rsa:4096 -nodes -batch -keyout client4-key.pem -out client4-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client4" +openssl req -newkey rsa:4096 -nodes -batch -keyout client5-key.pem -out client5-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client5" +openssl req -newkey rsa:4096 -nodes -batch -keyout client6-key.pem -out client6-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client6" # 5. Use CA's private key to sign client's CSR and get back the signed certificate openssl x509 -req -days 3650 -in client1-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client1-cert.pem openssl x509 -req -days 3650 -in client2-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client2-cert.pem openssl x509 -req -days 3650 -in client3-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client3-cert.pem openssl x509 -req -days 3650 -in client4-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile client4-ext.cnf -out client4-cert.pem +openssl x509 -req -days 3650 -in client5-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile client5-ext.cnf -out client5-cert.pem +openssl x509 -req -days 3650 -in client6-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile client6-ext.cnf -out client6-cert.pem # 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" diff --git a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml index 4bd30163ea6..b697c010195 100644 --- a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml +++ b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml @@ -17,6 +17,11 @@ URI:spiffe://foo.com/baz + + + URI:spiffe://bar.com/foo/*/far + + diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 3af88759e82..19ad67ec136 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -297,6 +297,8 @@ def test_https_non_ssl_auth(): def test_create_user(): + instance.query("DROP USER IF EXISTS emma") + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") assert ( execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") @@ -330,12 +332,16 @@ def test_create_user(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" ) - == 'emma\tssl_certificate\t{"common_names":["client2"]}\n' - 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' + == "emma\t['ssl_certificate']\t['{\"common_names\":[\"client2\"]}']\n" + 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) + instance.query("DROP USER IF EXISTS emma") + def test_x509_san_support(): + instance.query("DROP USER IF EXISTS jemma") + assert ( execute_query_native( instance, "SELECT currentUser()", user="jerome", cert_name="client4" @@ -350,7 +356,7 @@ def test_x509_san_support(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name='jerome'" ) - == 'jerome\tssl_certificate\t{"subject_alt_names":["URI:spiffe:\\\\/\\\\/foo.com\\\\/bar","URI:spiffe:\\\\/\\\\/foo.com\\\\/baz"]}\n' + == 'jerome\t[\'ssl_certificate\']\t[\'{"subject_alt_names":["URI:spiffe:\\\\/\\\\/foo.com\\\\/bar","URI:spiffe:\\\\/\\\\/foo.com\\\\/baz"]}\']\n' ) # user `jerome` is configured via xml config, but `show create` should work regardless. assert ( @@ -369,3 +375,42 @@ def test_x509_san_support(): instance.query("SHOW CREATE USER jemma") == "CREATE USER jemma IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://foo.com/bar\\', \\'URI:spiffe://foo.com/baz\\'\n" ) + + instance.query("DROP USER IF EXISTS jemma") + + +def test_x509_san_wildcard_support(): + assert ( + execute_query_native( + instance, "SELECT currentUser()", user="stewie", cert_name="client5" + ) + == "stewie\n" + ) + + assert ( + instance.query( + "SELECT name, auth_type, auth_params FROM system.users WHERE name='stewie'" + ) + == "stewie\t['ssl_certificate']\t['{\"subject_alt_names\":[\"URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far\"]}']\n" + ) + + assert ( + instance.query("SHOW CREATE USER stewie") + == "CREATE USER stewie IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://bar.com/foo/*/far\\'\n" + ) + + instance.query( + "CREATE USER brian IDENTIFIED WITH ssl_certificate SAN 'URI:spiffe://bar.com/foo/*/far'" + ) + + assert ( + execute_query_https("SELECT currentUser()", user="brian", cert_name="client6") + == "brian\n" + ) + + assert ( + instance.query("SHOW CREATE USER brian") + == "CREATE USER brian IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://bar.com/foo/*/far\\'\n" + ) + + instance.query("DROP USER brian") diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 9e3ee19179a..b75ad21f002 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -662,10 +662,7 @@ def test_multiple_tables_meta_mismatch(started_cluster): }, ) except QueryRuntimeException as e: - assert ( - "Table columns structure in ZooKeeper is different from local table structure" - in str(e) - ) + assert "Existing table metadata in ZooKeeper differs in columns" in str(e) failed = True assert failed is True diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py index e36989a9cdb..f88ff5da814 100644 --- a/tests/integration/test_tlsv1_3/test.py +++ b/tests/integration/test_tlsv1_3/test.py @@ -186,6 +186,8 @@ def test_https_non_ssl_auth(): def test_create_user(): + instance.query("DROP USER IF EXISTS emma") + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") assert ( execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") @@ -219,6 +221,8 @@ def test_create_user(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" ) - == 'emma\tssl_certificate\t{"common_names":["client2"]}\n' - 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' + == "emma\t['ssl_certificate']\t['{\"common_names\":[\"client2\"]}']\n" + 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) + + instance.query("DROP USER IF EXISTS emma") diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py index 39ca5997067..50b7dd098c9 100644 --- a/tests/integration/test_user_valid_until/test.py +++ b/tests/integration/test_user_valid_until/test.py @@ -19,10 +19,15 @@ def started_cluster(): def test_basic(started_cluster): + node.query("DROP USER IF EXISTS user_basic") + # 1. Without VALID UNTIL node.query("CREATE USER user_basic") - assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic\n" + assert ( + node.query("SHOW CREATE USER user_basic") + == "CREATE USER user_basic IDENTIFIED WITH no_password\n" + ) assert node.query("SELECT 1", user="user_basic") == "1\n" # 2. With valid VALID UNTIL @@ -30,7 +35,7 @@ def test_basic(started_cluster): assert ( node.query("SHOW CREATE USER user_basic") - == "CREATE USER user_basic VALID UNTIL \\'2040-11-06 05:03:20\\'\n" + == "CREATE USER user_basic IDENTIFIED WITH no_password VALID UNTIL \\'2040-11-06 05:03:20\\'\n" ) assert node.query("SELECT 1", user="user_basic") == "1\n" @@ -39,7 +44,7 @@ def test_basic(started_cluster): assert ( node.query("SHOW CREATE USER user_basic") - == "CREATE USER user_basic VALID UNTIL \\'2010-11-06 05:03:20\\'\n" + == "CREATE USER user_basic IDENTIFIED WITH no_password VALID UNTIL \\'2010-11-06 05:03:20\\'\n" ) error = "Authentication failed" @@ -48,7 +53,10 @@ def test_basic(started_cluster): # 4. Reset VALID UNTIL node.query("ALTER USER user_basic VALID UNTIL 'infinity'") - assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic\n" + assert ( + node.query("SHOW CREATE USER user_basic") + == "CREATE USER user_basic IDENTIFIED WITH no_password\n" + ) assert node.query("SELECT 1", user="user_basic") == "1\n" node.query("DROP USER user_basic") @@ -65,41 +73,53 @@ def test_basic(started_cluster): error = "Authentication failed" assert error in node.query_and_get_error("SELECT 1", user="user_basic") + node.query("DROP USER IF EXISTS user_basic") + def test_details(started_cluster): + node.query("DROP USER IF EXISTS user_details_infinity, user_details_time_only") + # 1. Does not do anything node.query("CREATE USER user_details_infinity VALID UNTIL 'infinity'") assert ( node.query("SHOW CREATE USER user_details_infinity") - == "CREATE USER user_details_infinity\n" + == "CREATE USER user_details_infinity IDENTIFIED WITH no_password\n" ) # 2. Time only is not supported - node.query("CREATE USER user_details_time_only VALID UNTIL '22:03:40'") + node.query( + "CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL '22:03:40'" + ) until_year = datetime.today().strftime("%Y") assert ( node.query("SHOW CREATE USER user_details_time_only") - == f"CREATE USER user_details_time_only VALID UNTIL \\'{until_year}-01-01 22:03:40\\'\n" + == f"CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL \\'{until_year}-01-01 22:03:40\\'\n" ) + node.query("DROP USER IF EXISTS user_details_infinity, user_details_time_only") + def test_restart(started_cluster): + node.query("DROP USER IF EXISTS user_restart") + node.query("CREATE USER user_restart VALID UNTIL '06/11/2010 08:03:20 Z+3'") assert ( node.query("SHOW CREATE USER user_restart") - == "CREATE USER user_restart VALID UNTIL \\'2010-11-06 05:03:20\\'\n" + == "CREATE USER user_restart IDENTIFIED WITH no_password VALID UNTIL \\'2010-11-06 05:03:20\\'\n" ) node.restart_clickhouse() assert ( node.query("SHOW CREATE USER user_restart") - == "CREATE USER user_restart VALID UNTIL \\'2010-11-06 05:03:20\\'\n" + == "CREATE USER user_restart IDENTIFIED WITH no_password VALID UNTIL \\'2010-11-06 05:03:20\\'\n" ) error = "Authentication failed" assert error in node.query_and_get_error("SELECT 1", user="user_restart") + + node.query("DROP USER IF EXISTS user_restart") diff --git a/tests/integration/test_zero_copy_unfreeze/__init__.py b/tests/integration/test_zero_copy_unfreeze/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_zero_copy_unfreeze/configs/storage_conf.xml b/tests/integration/test_zero_copy_unfreeze/configs/storage_conf.xml new file mode 100644 index 00000000000..683ff7055a4 --- /dev/null +++ b/tests/integration/test_zero_copy_unfreeze/configs/storage_conf.xml @@ -0,0 +1,42 @@ + + + test + + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + encrypted + s3 + 9bb21f41535d8e89098f3f23a62c661e + + + + + +

+ s3 +
+ + + + +
+ s3 +
+ + + + + + true + false + + true + diff --git a/tests/integration/test_zero_copy_unfreeze/test.py b/tests/integration/test_zero_copy_unfreeze/test.py new file mode 100644 index 00000000000..9c1902c97ee --- /dev/null +++ b/tests/integration/test_zero_copy_unfreeze/test.py @@ -0,0 +1,58 @@ +from collections.abc import Iterable +import pytest + +from helpers.cluster import ClickHouseCluster, ClickHouseInstance + +cluster = ClickHouseCluster(__file__) + + +@pytest.fixture(scope="module") +def started_cluster() -> Iterable[ClickHouseCluster]: + try: + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml"], + with_minio=True, + with_zookeeper=True, + ) + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +@pytest.mark.parametrize("storage_policy", ["s3", "encrypted"]) +def test_unfreeze(storage_policy: str, started_cluster: ClickHouseCluster) -> None: + node1: ClickHouseInstance = started_cluster.instances["node1"] + node1.query( + f"""\ +CREATE TABLE test1 (a Int) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test1', 'r1') +ORDER BY a +SETTINGS storage_policy = '{storage_policy}' +""" + ) + + node1.query( + """\ +INSERT INTO test1 +SELECT * +FROM system.numbers +LIMIT 20 +""" + ) + + node1.query("ALTER TABLE test1 FREEZE WITH NAME 'test'") + node1.query("SYSTEM UNFREEZE WITH NAME 'test'") + uuid = node1.query("SELECT uuid FROM system.tables WHERE name = 'test1'").strip() + # ensure that zero copy lock parent still exists + kazoo = started_cluster.get_kazoo_client("zoo1") + part_path = f"/clickhouse/zero_copy/zero_copy_s3/{uuid}/all_0_0_0/" + children: list[str] = kazoo.get_children(part_path) + assert len(children) == 1 + part_name = children[0] + assert len(kazoo.get_children(part_path + part_name)) == 1 + assert node1.query("SELECT count() FROM test1").strip() == "20" + node1.query("DROP TABLE test1") + node1.query("SYSTEM DROP REPLICA 'r1' FROM ZKPATH '/clickhouse-tables/test1'") diff --git a/tests/performance/scripts/compare.sh b/tests/performance/scripts/compare.sh index da7bbf77a28..c3566c51a16 100755 --- a/tests/performance/scripts/compare.sh +++ b/tests/performance/scripts/compare.sh @@ -427,7 +427,7 @@ do done # for each query run, prepare array of metrics from query log -clickhouse-local --multiquery --query " +clickhouse-local --query " create view query_runs as select * from file('analyze/query-runs.tsv', TSV, 'test text, query_index int, query_id text, version UInt8, time float'); @@ -582,7 +582,7 @@ numactl --cpunodebind=all --membind=all numactl --show # If the available memory falls below 2 * size, GNU parallel will suspend some of the running jobs. numactl --cpunodebind=all --membind=all parallel -v --joblog analyze/parallel-log.txt --memsuspend 15G --null < analyze/commands.txt 2>> analyze/errors.log -clickhouse-local --multiquery --query " +clickhouse-local --query " -- Join the metric names back to the metric statistics we've calculated, and make -- a denormalized table of them -- statistics for all metrics for all queries. -- The WITH, ARRAY JOIN and CROSS JOIN do not like each other: @@ -680,7 +680,7 @@ rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.ts cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: -clickhouse-local --multiquery --query " +clickhouse-local --query " create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -981,7 +981,7 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts for version in {right,left} do rm -rf data - clickhouse-local --multiquery --query " + clickhouse-local --query " create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1151,7 +1151,7 @@ function report_metrics rm -rf metrics ||: mkdir metrics -clickhouse-local --multiquery --query " +clickhouse-local --query " create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes) ; @@ -1211,7 +1211,7 @@ function upload_results # Prepare info for the CI checks table. rm -f ci-checks.tsv - clickhouse-local --multiquery --query " + clickhouse-local --query " create view queries as select * from file('report/queries.tsv', TSVWithNamesAndTypes); create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv') diff --git a/tests/queries/0_stateless/01045_array_zip.reference b/tests/queries/0_stateless/01045_array_zip.reference index 955ed98033e..154afa7eb89 100644 --- a/tests/queries/0_stateless/01045_array_zip.reference +++ b/tests/queries/0_stateless/01045_array_zip.reference @@ -1,2 +1,3 @@ [('a','d'),('b','e'),('c','f')] [('a','d','g'),('b','e','h'),('c','f','i')] +[] diff --git a/tests/queries/0_stateless/01045_array_zip.sql b/tests/queries/0_stateless/01045_array_zip.sql index 0bf77747123..801df5a3230 100644 --- a/tests/queries/0_stateless/01045_array_zip.sql +++ b/tests/queries/0_stateless/01045_array_zip.sql @@ -2,7 +2,7 @@ SELECT arrayZip(['a', 'b', 'c'], ['d', 'e', 'f']); SELECT arrayZip(['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']); -SELECT arrayZip(); -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION } +SELECT arrayZip(); SELECT arrayZip('a', 'b', 'c'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.reference b/tests/queries/0_stateless/01073_grant_and_revoke.reference index b91820914e6..b2c7797ad3c 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.reference +++ b/tests/queries/0_stateless/01073_grant_and_revoke.reference @@ -1,5 +1,5 @@ A -CREATE USER test_user_01073 +CREATE USER test_user_01073 IDENTIFIED WITH no_password B C GRANT INSERT, ALTER DELETE ON *.* TO test_user_01073 diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.sql b/tests/queries/0_stateless/01073_grant_and_revoke.sql index 4cffd916e9f..59f599ce140 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.sql +++ b/tests/queries/0_stateless/01073_grant_and_revoke.sql @@ -1,3 +1,5 @@ +-- Tags: no-parallel + DROP USER IF EXISTS test_user_01073; DROP ROLE IF EXISTS test_role_01073; diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.reference b/tests/queries/0_stateless/01075_allowed_client_hosts.reference index 5fb11bae65e..5d2168d6c3b 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.reference +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.reference @@ -1,17 +1,17 @@ -CREATE USER test_user_01075 -CREATE USER test_user_01075 -CREATE USER test_user_01075 HOST NONE -CREATE USER test_user_01075 HOST LOCAL -CREATE USER test_user_01075 HOST IP \'192.168.23.15\' -CREATE USER test_user_01075 HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' -CREATE USER test_user_01075 HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' -CREATE USER test_user_01075 HOST LOCAL -CREATE USER test_user_01075 HOST NONE -CREATE USER test_user_01075 HOST LIKE \'@.somesite.com\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\.anothersite\\\\.org\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\' -CREATE USER `test_user_01075_x@localhost` HOST LOCAL -CREATE USER test_user_01075_x HOST LOCAL -CREATE USER `test_user_01075_x@192.168.23.15` HOST LOCAL +CREATE USER test_user_01075 IDENTIFIED WITH no_password +CREATE USER test_user_01075 IDENTIFIED WITH no_password +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST NONE +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST IP \'192.168.23.15\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST NONE +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LIKE \'@.somesite.com\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite\\\\.com\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\.anothersite\\\\.org\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\' +CREATE USER `test_user_01075_x@localhost` IDENTIFIED WITH no_password HOST LOCAL +CREATE USER test_user_01075_x IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `test_user_01075_x@192.168.23.15` IDENTIFIED WITH no_password HOST LOCAL diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.sql b/tests/queries/0_stateless/01075_allowed_client_hosts.sql index 17957c8826b..8c25d45f421 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.sql +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-parallel DROP USER IF EXISTS test_user_01075, test_user_01075_x, test_user_01075_x@localhost, test_user_01075_x@'192.168.23.15'; diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index d5841a74a2c..3df69ae2669 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -1,12 +1,12 @@ -- default -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password -- same as default -CREATE USER u2_01292 -CREATE USER u3_01292 +CREATE USER u2_01292 IDENTIFIED WITH no_password +CREATE USER u3_01292 IDENTIFIED WITH no_password -- rename -CREATE USER u2_01292_renamed +CREATE USER u2_01292_renamed IDENTIFIED WITH no_password -- authentication -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password CREATE USER u2_01292 IDENTIFIED WITH plaintext_password CREATE USER u3_01292 IDENTIFIED WITH sha256_password CREATE USER u4_01292 IDENTIFIED WITH sha256_password @@ -19,97 +19,97 @@ CREATE USER u1_01292 IDENTIFIED WITH sha256_password CREATE USER u2_01292 IDENTIFIED WITH sha256_password CREATE USER u3_01292 IDENTIFIED WITH sha256_password CREATE USER u4_01292 IDENTIFIED WITH plaintext_password -CREATE USER u5_01292 +CREATE USER u5_01292 IDENTIFIED WITH no_password -- host -CREATE USER u1_01292 -CREATE USER u2_01292 HOST NONE -CREATE USER u3_01292 HOST LOCAL -CREATE USER u4_01292 HOST NAME \'myhost.com\' -CREATE USER u5_01292 HOST LOCAL, NAME \'myhost.com\' -CREATE USER u6_01292 HOST LOCAL, NAME \'myhost.com\' -CREATE USER u7_01292 HOST REGEXP \'.*\\\\.myhost\\\\.com\' -CREATE USER u8_01292 -CREATE USER u9_01292 HOST LIKE \'%.myhost.com\' -CREATE USER u10_01292 HOST LIKE \'%.myhost.com\' -CREATE USER u11_01292 HOST LOCAL -CREATE USER u12_01292 HOST IP \'192.168.1.1\' -CREATE USER u13_01292 HOST IP \'192.168.0.0/16\' -CREATE USER u14_01292 HOST LOCAL -CREATE USER u15_01292 HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' -CREATE USER u16_01292 HOST LOCAL, IP \'65:ff0c::/96\' -CREATE USER u1_01292 HOST NONE -CREATE USER u2_01292 HOST NAME \'myhost.com\' -CREATE USER u3_01292 HOST LOCAL, NAME \'myhost.com\' -CREATE USER u4_01292 HOST NONE +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password HOST NONE +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST NAME \'myhost.com\' +CREATE USER u5_01292 IDENTIFIED WITH no_password HOST LOCAL, NAME \'myhost.com\' +CREATE USER u6_01292 IDENTIFIED WITH no_password HOST LOCAL, NAME \'myhost.com\' +CREATE USER u7_01292 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.myhost\\\\.com\' +CREATE USER u8_01292 IDENTIFIED WITH no_password +CREATE USER u9_01292 IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER u10_01292 IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER u11_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER u12_01292 IDENTIFIED WITH no_password HOST IP \'192.168.1.1\' +CREATE USER u13_01292 IDENTIFIED WITH no_password HOST IP \'192.168.0.0/16\' +CREATE USER u14_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER u15_01292 IDENTIFIED WITH no_password HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER u16_01292 IDENTIFIED WITH no_password HOST LOCAL, IP \'65:ff0c::/96\' +CREATE USER u1_01292 IDENTIFIED WITH no_password HOST NONE +CREATE USER u2_01292 IDENTIFIED WITH no_password HOST NAME \'myhost.com\' +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LOCAL, NAME \'myhost.com\' +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST NONE -- host after @ -CREATE USER u1_01292 -CREATE USER u1_01292 -CREATE USER `u2_01292@%.myhost.com` HOST LIKE \'%.myhost.com\' -CREATE USER `u2_01292@%.myhost.com` HOST LIKE \'%.myhost.com\' -CREATE USER `u3_01292@192.168.%.%` HOST LIKE \'192.168.%.%\' -CREATE USER `u3_01292@192.168.%.%` HOST LIKE \'192.168.%.%\' -CREATE USER `u4_01292@::1` HOST LOCAL -CREATE USER `u4_01292@::1` HOST LOCAL -CREATE USER `u5_01292@65:ff0c::/96` HOST LIKE \'65:ff0c::/96\' -CREATE USER `u5_01292@65:ff0c::/96` HOST LIKE \'65:ff0c::/96\' -CREATE USER u1_01292 HOST LOCAL -CREATE USER `u2_01292@%.myhost.com` +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER `u2_01292@%.myhost.com` IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER `u2_01292@%.myhost.com` IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER `u3_01292@192.168.%.%` IDENTIFIED WITH no_password HOST LIKE \'192.168.%.%\' +CREATE USER `u3_01292@192.168.%.%` IDENTIFIED WITH no_password HOST LIKE \'192.168.%.%\' +CREATE USER `u4_01292@::1` IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `u4_01292@::1` IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `u5_01292@65:ff0c::/96` IDENTIFIED WITH no_password HOST LIKE \'65:ff0c::/96\' +CREATE USER `u5_01292@65:ff0c::/96` IDENTIFIED WITH no_password HOST LIKE \'65:ff0c::/96\' +CREATE USER u1_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `u2_01292@%.myhost.com` IDENTIFIED WITH no_password -- settings -CREATE USER u1_01292 -CREATE USER u2_01292 SETTINGS PROFILE `default` -CREATE USER u3_01292 SETTINGS max_memory_usage = 5000000 -CREATE USER u4_01292 SETTINGS max_memory_usage MIN 5000000 -CREATE USER u5_01292 SETTINGS max_memory_usage MAX 5000000 -CREATE USER u6_01292 SETTINGS max_memory_usage CONST -CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE -CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST -CREATE USER u9_01292 SETTINGS PROFILE `default`, max_memory_usage = 5000000 WRITABLE -CREATE USER u1_01292 SETTINGS readonly = 1 -CREATE USER u2_01292 SETTINGS readonly = 1 -CREATE USER u3_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password SETTINGS PROFILE `default` +CREATE USER u3_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage = 5000000 +CREATE USER u4_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage MIN 5000000 +CREATE USER u5_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage MAX 5000000 +CREATE USER u6_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage CONST +CREATE USER u7_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage WRITABLE +CREATE USER u8_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST +CREATE USER u9_01292 IDENTIFIED WITH no_password SETTINGS PROFILE `default`, max_memory_usage = 5000000 WRITABLE +CREATE USER u1_01292 IDENTIFIED WITH no_password SETTINGS readonly = 1 +CREATE USER u2_01292 IDENTIFIED WITH no_password SETTINGS readonly = 1 +CREATE USER u3_01292 IDENTIFIED WITH no_password -- default role -CREATE USER u1_01292 -CREATE USER u2_01292 DEFAULT ROLE NONE -CREATE USER u3_01292 DEFAULT ROLE r1_01292 -CREATE USER u4_01292 DEFAULT ROLE r1_01292, r2_01292 -CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r2_01292 -CREATE USER u6_01292 DEFAULT ROLE ALL EXCEPT r1_01292, r2_01292 -CREATE USER u1_01292 DEFAULT ROLE r1_01292 -CREATE USER u2_01292 DEFAULT ROLE ALL EXCEPT r2_01292 -CREATE USER u3_01292 DEFAULT ROLE r2_01292 -CREATE USER u4_01292 -CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r1_01292 -CREATE USER u6_01292 DEFAULT ROLE NONE +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE +CREATE USER u3_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292 +CREATE USER u4_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u5_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r2_01292 +CREATE USER u6_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r1_01292, r2_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292 +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r2_01292 +CREATE USER u3_01292 IDENTIFIED WITH no_password DEFAULT ROLE r2_01292 +CREATE USER u4_01292 IDENTIFIED WITH no_password +CREATE USER u5_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r1_01292 +CREATE USER u6_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE -- complex CREATE USER u1_01292 IDENTIFIED WITH plaintext_password HOST LOCAL SETTINGS readonly = 1 -CREATE USER u1_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE NONE SETTINGS PROFILE `default` +CREATE USER u1_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE NONE SETTINGS PROFILE `default` -- if not exists -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password GRANT r1_01292 TO u1_01292 -- if not exists-part2 -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password GRANT r1_01292, r2_01292 TO u1_01292 -- or replace -CREATE USER u1_01292 -CREATE USER u2_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password -- multiple users in one command -CREATE USER u1_01292 DEFAULT ROLE NONE -CREATE USER u2_01292 DEFAULT ROLE NONE -CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' -CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' -CREATE USER `u5_01292@%.host.com` HOST LIKE \'%.host.com\' -CREATE USER `u6_01292@%.host.com` HOST LIKE \'%.host.com\' -CREATE USER `u7_01292@%.host.com` HOST LIKE \'%.host.com\' -CREATE USER `u8_01292@%.otherhost.com` HOST LIKE \'%.otherhost.com\' -CREATE USER u1_01292 DEFAULT ROLE NONE SETTINGS readonly = 1 -CREATE USER u2_01292 DEFAULT ROLE r1_01292, r2_01292 SETTINGS readonly = 1 -CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 -CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' +CREATE USER `u5_01292@%.host.com` IDENTIFIED WITH no_password HOST LIKE \'%.host.com\' +CREATE USER `u6_01292@%.host.com` IDENTIFIED WITH no_password HOST LIKE \'%.host.com\' +CREATE USER `u7_01292@%.host.com` IDENTIFIED WITH no_password HOST LIKE \'%.host.com\' +CREATE USER `u8_01292@%.otherhost.com` IDENTIFIED WITH no_password HOST LIKE \'%.otherhost.com\' +CREATE USER u1_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE SETTINGS readonly = 1 +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292, r2_01292 SETTINGS readonly = 1 +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 -- system.users -u1_01292 local_directory plaintext_password {} [] ['localhost'] [] [] 1 [] [] -u2_01292 local_directory no_password {} [] [] [] ['%.%.myhost.com'] 0 [] [] -u3_01292 local_directory sha256_password {} ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] -u4_01292 local_directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_01292'] +u1_01292 local_directory ['plaintext_password'] ['{}'] [] ['localhost'] [] [] 1 [] [] +u2_01292 local_directory ['no_password'] ['{}'] [] [] [] ['%.%.myhost.com'] 0 [] [] +u3_01292 local_directory ['sha256_password'] ['{}'] ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] +u4_01292 local_directory ['double_sha1_password'] ['{}'] ['::/0'] [] [] [] 1 [] ['r1_01292'] -- system.settings_profile_elements \N u1_01292 \N 0 readonly 1 \N \N \N \N \N u2_01292 \N 0 \N \N \N \N \N default @@ -117,3 +117,5 @@ u4_01292 local_directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_012 \N u4_01292 \N 0 \N \N \N \N \N default \N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N \N u4_01292 \N 2 readonly 1 \N \N \N \N +-- multiple authentication methods +u1_01292 ['plaintext_password','kerberos','bcrypt_password','ldap'] ['{}','{"realm":"qwerty10"}','{}','{"server":"abc"}'] diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql index 46808aec1ef..41633b3d423 100644 --- a/tests/queries/0_stateless/01292_create_user.sql +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -233,3 +233,8 @@ SELECT * FROM system.settings_profile_elements WHERE user_name LIKE 'u%\_01292' DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292; DROP ROLE r1_01292, r2_01292; + +SELECT '-- multiple authentication methods'; +CREATE USER u1_01292 IDENTIFIED WITH plaintext_password by '1', kerberos REALM 'qwerty10', bcrypt_password by '3', ldap SERVER 'abc'; +SELECT name, auth_type, auth_params FROM system.users WHERE name = 'u1_01292' ORDER BY name; +DROP USER u1_01292; diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference index 48d8b4ee8a1..72e0dd9fb50 100644 --- a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference @@ -1 +1 @@ -CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' +CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' diff --git a/tests/queries/0_stateless/01801_s3_cluster.reference b/tests/queries/0_stateless/01801_s3_cluster.reference index 4166d1718b1..c77baca9f09 100644 --- a/tests/queries/0_stateless/01801_s3_cluster.reference +++ b/tests/queries/0_stateless/01801_s3_cluster.reference @@ -190,3 +190,195 @@ 20 21 22 23 24 25 26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 diff --git a/tests/queries/0_stateless/01801_s3_cluster.sql b/tests/queries/0_stateless/01801_s3_cluster.sql index 68d90ea4be0..f94f1102dc0 100644 --- a/tests/queries/0_stateless/01801_s3_cluster.sql +++ b/tests/queries/0_stateless/01801_s3_cluster.sql @@ -2,21 +2,37 @@ -- Tag no-fasttest: Depends on AWS select * from s3('http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; -select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; -select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; diff --git a/tests/queries/0_stateless/01939_user_with_default_database.reference b/tests/queries/0_stateless/01939_user_with_default_database.reference index 8c8ff7e3007..6e5a1d20758 100644 --- a/tests/queries/0_stateless/01939_user_with_default_database.reference +++ b/tests/queries/0_stateless/01939_user_with_default_database.reference @@ -1,4 +1,4 @@ default db_01939 -CREATE USER u_01939 -CREATE USER u_01939 DEFAULT DATABASE NONE +CREATE USER u_01939 IDENTIFIED WITH no_password +CREATE USER u_01939 IDENTIFIED WITH no_password DEFAULT DATABASE NONE diff --git a/tests/queries/0_stateless/01999_grant_with_replace.reference b/tests/queries/0_stateless/01999_grant_with_replace.reference index dc2047ab73c..903f2c301a0 100644 --- a/tests/queries/0_stateless/01999_grant_with_replace.reference +++ b/tests/queries/0_stateless/01999_grant_with_replace.reference @@ -1,4 +1,4 @@ -CREATE USER test_user_01999 +CREATE USER test_user_01999 IDENTIFIED WITH no_password A B GRANT SELECT ON db1.* TO test_user_01999 diff --git a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh index 972ff3ba73f..7bf8100a53d 100755 --- a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh +++ b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh @@ -5,4 +5,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' +$CLICKHOUSE_CLIENT -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' diff --git a/tests/queries/0_stateless/02030_rocksdb_race_long.sh b/tests/queries/0_stateless/02030_rocksdb_race_long.sh index da31861991c..7d329f78d98 100755 --- a/tests/queries/0_stateless/02030_rocksdb_race_long.sh +++ b/tests/queries/0_stateless/02030_rocksdb_race_long.sh @@ -12,14 +12,14 @@ echo " DROP TABLE IF EXISTS rocksdb_race; CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000); -" | $CLICKHOUSE_CLIENT -n +" | $CLICKHOUSE_CLIENT function read_stat_thread() { while true; do echo " SELECT * FROM system.rocksdb FORMAT Null; - " | $CLICKHOUSE_CLIENT -n + " | $CLICKHOUSE_CLIENT done } @@ -29,7 +29,7 @@ function truncate_thread() sleep 3s; echo " TRUNCATE TABLE rocksdb_race; - " | $CLICKHOUSE_CLIENT -n + " | $CLICKHOUSE_CLIENT done } diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 638a46a142f..e874687fd4c 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -1143,8 +1143,8 @@ CREATE TABLE system.users `name` String, `id` UUID, `storage` String, - `auth_type` Enum8('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6, 'bcrypt_password' = 7, 'ssh_key' = 8, 'http' = 9, 'jwt' = 10), - `auth_params` String, + `auth_type` Array(Enum8('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6, 'bcrypt_password' = 7, 'ssh_key' = 8, 'http' = 9, 'jwt' = 10)), + `auth_params` Array(String), `host_ip` Array(String), `host_names` Array(String), `host_names_regexp` Array(String), diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh index 007dae6e427..a9b5c461c5a 100755 --- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh +++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh @@ -12,7 +12,7 @@ opts=( --join_algorithm='parallel_hash' ) -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " CREATE TABLE t1(a UInt32, b UInt32) ENGINE=MergeTree ORDER BY (); INSERT INTO t1 SELECT number, number FROM numbers_mt(1e6); diff --git a/tests/queries/0_stateless/02156_storage_merge_prewhere.reference b/tests/queries/0_stateless/02156_storage_merge_prewhere.reference index 876cee60baa..8aa1f0b59d3 100644 --- a/tests/queries/0_stateless/02156_storage_merge_prewhere.reference +++ b/tests/queries/0_stateless/02156_storage_merge_prewhere.reference @@ -1,25 +1,25 @@ Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) 2 Filter column: and(equals(k, 3), notEmpty(v)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) 2 Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) Prewhere info Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) 2 diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh index e5d00bc1a1c..b23164f8354 100755 --- a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -5,12 +5,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -nm -q " +timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -m -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " -timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -nm -q " +timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -m -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh index f8e7b7e7e72..8a44b65e6d1 100755 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh @@ -16,7 +16,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM DROP FILESYSTEM CACHE" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -37,7 +37,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -70,7 +70,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -109,7 +109,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM FLUSH LOGS" - $CLICKHOUSE_CLIENT -n --query "SELECT + $CLICKHOUSE_CLIENT --query "SELECT query, ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read FROM system.query_log diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh index 2af54276469..3cb79d69955 100755 --- a/tests/queries/0_stateless/02242_delete_user_race.sh +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -15,7 +15,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP ROLE IF EXISTS test_role_02242; CREATE ROLE test_role_02242; " diff --git a/tests/queries/0_stateless/02243_drop_user_grant_race.sh b/tests/queries/0_stateless/02243_drop_user_grant_race.sh index 4dce8e8124c..1972ecade06 100755 --- a/tests/queries/0_stateless/02243_drop_user_grant_race.sh +++ b/tests/queries/0_stateless/02243_drop_user_grant_race.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP ROLE IF EXISTS test_role_02244; CREATE ROLE test_role_02244; DROP USER IF EXISTS kek_02243; @@ -37,4 +37,4 @@ $CLICKHOUSE_CLIENT --user kek_02243 -q "SELECT * FROM test" 2>&1| grep -Fa "Exce $CLICKHOUSE_CLIENT -q "DROP ROLE IF EXISTS test_role_02243" $CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_02243" -$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" diff --git a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh index ae2a2351c6b..eb1109aaac1 100755 --- a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh +++ b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh @@ -44,7 +44,7 @@ protobuf_info() { fi } -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS $MAIN_TABLE; DROP TABLE IF EXISTS $ROUNDTRIP_TABLE; DROP TABLE IF EXISTS $COMPATIBILITY_TABLE; @@ -78,14 +78,14 @@ echo $SET_OUTPUT echo echo "Insert $INITIAL_INSERT_VALUES into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " INSERT INTO $MAIN_TABLE VALUES $INITIAL_INSERT_VALUES; SELECT * FROM $MAIN_TABLE; " echo echo "Protobuf representation of the second row:" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -101,12 +101,12 @@ hexdump -C $MESSAGE_FILE_PATH echo echo "Insert proto message into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $ROUNDTRIP_TABLE" echo echo "Proto output of the table using Google wrapper:" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -124,14 +124,14 @@ echo echo "Insert $MULTI_WRAPPER_VALUES and reinsert using Google wrappers into:" echo "Table (Nullable(Int32), Nullable(Int32), Int32):" $CLICKHOUSE_CLIENT --query "INSERT INTO $MULTI_TABLE VALUES $MULTI_WRAPPER_VALUES" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" -$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $MULTI_TABLE" rm "$BINARY_FILE_PATH" rm "$MESSAGE_FILE_PATH" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE $MAIN_TABLE; DROP TABLE $ROUNDTRIP_TABLE; DROP TABLE $COMPATIBILITY_TABLE; diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index dab777fcc31..672bf4d068b 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -11,7 +11,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286" - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_02286 (key UInt32, value String) + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_02286 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760" @@ -38,7 +38,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null" $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT -n --query "SELECT count() + $CLICKHOUSE_CLIENT --query "SELECT count() FROM ( SELECT arrayJoin(cache_paths) AS cache_path, @@ -54,7 +54,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" $CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT -n --query "SELECT cache_path, local_path + $CLICKHOUSE_CLIENT --query "SELECT cache_path, local_path FROM ( SELECT arrayJoin(cache_paths) AS cache_path, diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh index b7adde6fcbb..64fae06f220 100755 --- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh +++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh @@ -23,7 +23,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d ORDER BY tuple() SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null - $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 -n --query "INSERT INTO test_02313 + $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02313 SELECT * FROM generateRandom('id Int32, val String') LIMIT 100000" diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index bd018018789..f252b9304cd 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -9,7 +9,7 @@ function check_refcnt_for_table() { local table=$1 && shift - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " system stop merges $table; -- cleanup thread may hold the parts lock system stop cleanup $table; @@ -66,14 +66,14 @@ function check_refcnt_for_table() # NOTE: index_granularity=1 to cancel ASAP -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " drop table if exists data_02340; create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1; " || exit 1 check_refcnt_for_table data_02340 $CLICKHOUSE_CLIENT -q "drop table data_02340 sync" -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " drop table if exists data_02340_rep sync; create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1; " || exit 1 diff --git a/tests/queries/0_stateless/02344_describe_cache.sh b/tests/queries/0_stateless/02344_describe_cache.sh index c5373b4d7e3..4cc4c415f3f 100755 --- a/tests/queries/0_stateless/02344_describe_cache.sh +++ b/tests/queries/0_stateless/02344_describe_cache.sh @@ -7,14 +7,14 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02344_describe_cache_test" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = '$disk_name', disk = 's3_disk', load_metadata_asynchronously = 0); """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02346_inverted_index_experimental_flag.sql b/tests/queries/0_stateless/02346_inverted_index_experimental_flag.sql index f2d294ff9e4..84188337a8d 100644 --- a/tests/queries/0_stateless/02346_inverted_index_experimental_flag.sql +++ b/tests/queries/0_stateless/02346_inverted_index_experimental_flag.sql @@ -1,16 +1,60 @@ --- Tests that the inverted index can only be supported when allow_experimental_full_text_index = 1. - -SET allow_experimental_full_text_index = 0; +-- Tests that CREATE TABLE and ADD INDEX respect settings 'allow_experimental_full_text_index' and `allow_experimental_inverted_index` DROP TABLE IF EXISTS tab; -CREATE TABLE tab -( - `key` UInt64, - `str` String -) -ENGINE = MergeTree -ORDER BY key; -ALTER TABLE tab ADD INDEX inv_idx(str) TYPE full_text(0); -- { serverError SUPPORT_IS_DISABLED } +-- Test CREATE TABLE + full_text index setting +SET allow_experimental_full_text_index = 0; +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE full_text(0)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError SUPPORT_IS_DISABLED } +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE inverted(0)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError ILLEGAL_INDEX } + +SET allow_experimental_full_text_index = 1; +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE full_text(0)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE inverted(0)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError ILLEGAL_INDEX } DROP TABLE tab; + +SET allow_experimental_full_text_index = 0; -- reset to default + +-- Test CREATE TABLE + inverted index setting + +SET allow_experimental_inverted_index = 0; +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE full_text(0)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError SUPPORT_IS_DISABLED } +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE inverted(0)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError ILLEGAL_INDEX } + +SET allow_experimental_inverted_index = 1; +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE full_text(0)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError SUPPORT_IS_DISABLED } +CREATE TABLE tab (id UInt32, str String, INDEX idx str TYPE inverted(0)) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE tab; + +SET allow_experimental_inverted_index = 0; -- reset to default + +-- Test ADD INDEX + full_text index setting + +SET allow_experimental_full_text_index = 0; +CREATE TABLE tab (id UInt32, str String) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE tab ADD INDEX idx1 str TYPE full_text(0); -- { serverError SUPPORT_IS_DISABLED } +ALTER TABLE tab ADD INDEX idx2 str TYPE inverted(0); -- { serverError SUPPORT_IS_DISABLED } +DROP TABLE tab; + +SET allow_experimental_full_text_index = 1; +CREATE TABLE tab (id UInt32, str String) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE tab ADD INDEX idx1 str TYPE full_text(0); +ALTER TABLE tab ADD INDEX idx2 str TYPE inverted(0); -- { serverError SUPPORT_IS_DISABLED } +DROP TABLE tab; +SET allow_experimental_full_text_index = 0; -- reset to default + + +-- Test ADD INDEX + inverted index setting + +SET allow_experimental_inverted_index = 0; +CREATE TABLE tab (id UInt32, str String) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE tab ADD INDEX idx1 str TYPE full_text(0); -- { serverError SUPPORT_IS_DISABLED } +ALTER TABLE tab ADD INDEX idx2 str TYPE inverted(0); -- { serverError SUPPORT_IS_DISABLED } +DROP TABLE tab; + +SET allow_experimental_inverted_index = 1; +CREATE TABLE tab (id UInt32, str String) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE tab ADD INDEX idx1 str TYPE full_text(0); -- { serverError SUPPORT_IS_DISABLED } +ALTER TABLE tab ADD INDEX idx2 str TYPE inverted(0); +DROP TABLE tab; +SET allow_experimental_inverted_index = 0; -- reset to default diff --git a/tests/queries/0_stateless/02352_rwlock.sh b/tests/queries/0_stateless/02352_rwlock.sh index b4a77e0b08a..7f02b3ee935 100755 --- a/tests/queries/0_stateless/02352_rwlock.sh +++ b/tests/queries/0_stateless/02352_rwlock.sh @@ -24,7 +24,7 @@ function wait_query_by_id_started() # wait for query to be started while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do if [ "$( - $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -nm -q " + $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -m -q " system flush logs; select count() from system.query_log @@ -52,7 +52,7 @@ $CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_ordinary Engine=Or # debug build on CI, so if this will happen, then DROP query will be # finished instantly, and to avoid flakiness we will retry in this case while :; do - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352; CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null(); " diff --git a/tests/queries/0_stateless/02380_insert_mv_race.sh b/tests/queries/0_stateless/02380_insert_mv_race.sh index 725c7eacce6..a5f05365180 100755 --- a/tests/queries/0_stateless/02380_insert_mv_race.sh +++ b/tests/queries/0_stateless/02380_insert_mv_race.sh @@ -9,13 +9,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q "ATTACH TABLE mv" |& { +$CLICKHOUSE_CLIENT -m -q "ATTACH TABLE mv" |& { # CANNOT_GET_CREATE_TABLE_QUERY -- ATTACH TABLE IF EXISTS # TABLE_ALREADY_EXISTS -- ATTACH TABLE IF NOT EXISTS grep -F -m1 Exception | grep -v -e CANNOT_GET_CREATE_TABLE_QUERY -e TABLE_ALREADY_EXISTS } -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS null; CREATE TABLE null (key Int) ENGINE = Null; DROP TABLE IF EXISTS mv; diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index 1368bf530c8..533389a40f6 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -560,7 +560,6 @@ positionCaseInsensitive positionCaseInsensitiveUTF8 positionUTF8 pow -printf proportionsZTest protocol queryID diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh index c692c37f27f..9e8b854375b 100755 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -27,7 +27,7 @@ function insert() function check_span() { -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS; SELECT operation_name, @@ -50,7 +50,7 @@ ${CLICKHOUSE_CLIENT} -nq " # $2 - value of distributed_foreground_insert function check_span_kind() { -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS; SELECT count() @@ -65,7 +65,7 @@ ${CLICKHOUSE_CLIENT} -nq " # # Prepare tables for tests # -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry; @@ -122,7 +122,7 @@ check_span_kind $trace_id 'CLIENT' # # Cleanup # -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry; " diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh index c43c5bb6408..6a23f666f36 100755 --- a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;" test_primary_key() { - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1); INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2); SELECT value FROM 02419_test WHERE key = 1; diff --git a/tests/queries/0_stateless/02443_detach_attach_partition.sh b/tests/queries/0_stateless/02443_detach_attach_partition.sh index 6a47b7d8d61..4a5377a952a 100755 --- a/tests/queries/0_stateless/02443_detach_attach_partition.sh +++ b/tests/queries/0_stateless/02443_detach_attach_partition.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/replication.lib -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; diff --git a/tests/queries/0_stateless/02447_drop_database_replica.sh b/tests/queries/0_stateless/02447_drop_database_replica.sh index abe99398a56..558a0c2693c 100755 --- a/tests/queries/0_stateless/02447_drop_database_replica.sh +++ b/tests/queries/0_stateless/02447_drop_database_replica.sh @@ -46,9 +46,13 @@ timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed # And that it still throws TIMEOUT_EXCEEDED for active replicas echo 'timeout on active' db9="${db}_9" -$CLICKHOUSE_CLIENT -q "create database $db9 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's9', 'r9')" +REPLICA_UUID=$($CLICKHOUSE_CLIENT -q "select serverUUID()") +if ! [[ $REPLICA_UUID =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]]; then + echo "Weird UUID ${REPLICA_UUID}" +fi +$CLICKHOUSE_CLIENT -q "create database $db9 engine=Replicated('/test/${CLICKHOUSE_DATABASE}/rdb', 's9', 'r9')" $CLICKHOUSE_CLIENT -q "detach database $db9" -$CLICKHOUSE_CLIENT -q "insert into system.zookeeper(name, path, value) values ('active', '/test/$CLICKHOUSE_DATABASE/rdb/replicas/s9|r9', '$($CLICKHOUSE_CLIENT -q "select serverUUID()")')" +$CLICKHOUSE_CLIENT -q "insert into system.zookeeper(name, path, value) values ('active', '/test/${CLICKHOUSE_DATABASE}/rdb/replicas/s9|r9', '${REPLICA_UUID}')" $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=none_only_active -q "create table $db.t22 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED" $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=throw_only_active -q "create table $db.t33 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED" diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.sh b/tests/queries/0_stateless/02473_optimize_old_parts.sh index b563bc31b39..3a4e6145f12 100755 --- a/tests/queries/0_stateless/02473_optimize_old_parts.sh +++ b/tests/queries/0_stateless/02473_optimize_old_parts.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_with_merge; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_without_merge; SELECT 'With merge any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_with_merge; SELECT 'With merge partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; diff --git a/tests/queries/0_stateless/02477_s3_request_throttler.sh b/tests/queries/0_stateless/02477_s3_request_throttler.sh index c74cb598d42..5e6185e7304 100755 --- a/tests/queries/0_stateless/02477_s3_request_throttler.sh +++ b/tests/queries/0_stateless/02477_s3_request_throttler.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " -- Limit S3 PUT request per second rate SET s3_max_put_rps = 2; SET s3_max_put_burst = 1; diff --git a/tests/queries/0_stateless/02494_query_cache_bugs.reference b/tests/queries/0_stateless/02494_query_cache_bugs.reference index 448e1366ea7..d50e9c42204 100644 --- a/tests/queries/0_stateless/02494_query_cache_bugs.reference +++ b/tests/queries/0_stateless/02494_query_cache_bugs.reference @@ -22,3 +22,4 @@ Row 1: ────── x: 1 2 +-- Bug 67476: Queries with overflow mode != throw must not be cached by the query cache diff --git a/tests/queries/0_stateless/02494_query_cache_bugs.sql b/tests/queries/0_stateless/02494_query_cache_bugs.sql index 74496e0f77a..755a5fae924 100644 --- a/tests/queries/0_stateless/02494_query_cache_bugs.sql +++ b/tests/queries/0_stateless/02494_query_cache_bugs.sql @@ -36,4 +36,22 @@ SELECT count(*) FROM system.query_cache; DROP TABLE tab; +SELECT '-- Bug 67476: Queries with overflow mode != throw must not be cached by the query cache'; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab(c UInt64) ENGINE = Memory; + +SYSTEM DROP QUERY CACHE; +SELECT sum(c) FROM tab SETTINGS read_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS read_overflow_mode_leaf = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS group_by_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS sort_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS result_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS timeout_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS set_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS join_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS transfer_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } +SELECT sum(c) FROM tab SETTINGS distinct_overflow_mode = 'break', use_query_cache = 1; -- { serverError QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE } + SYSTEM DROP QUERY CACHE; diff --git a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh index b72c3eb56c7..122684abe9c 100755 --- a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh +++ b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " @@ -60,7 +60,7 @@ wait $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat1 WHERE NOT ignore(*)" $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat2 WHERE NOT ignore(*)" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh index e389cf410e8..77c8049bd58 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -11,7 +11,7 @@ cp $CURDIR/data_ua_parser/os.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/browser.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/device.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; @@ -61,10 +61,10 @@ create table user_agents Engine = Log(); " -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" select ua, device, concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser , concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os @@ -74,7 +74,7 @@ from ( dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua; " -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh index 68a87a14320..dec03bcaaa0 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh @@ -27,7 +27,7 @@ cat > "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < /dev/null & diff --git a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh index 3b0d2309784..9f743581e1a 100755 --- a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh +++ b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh @@ -13,7 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function get_query_id() { random_str 10; } -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists buf; drop table if exists dist; drop table if exists data; @@ -31,7 +31,7 @@ query_id="$(get_query_id)" # test, since we care about the difference between NOW() and there should # not be any significant difference. $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " @@ -42,25 +42,25 @@ query_id="$(get_query_id)" # this query (and all subsequent) should reuse the previous connection (at least most of the time) $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " echo "INSERT" query_id="$(get_query_id)" -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -nm -q " +$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -m -q " insert into dist_dist values (1),(2); select * from data; " sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist_dist" +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist_dist" sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist" +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist" echo "CHECK" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " select * from data order by key; system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; diff --git a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh index ff534a6a2e6..5015521ca4d 100755 --- a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh +++ b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh echo "INSERT TO S3" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/profile_events.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10, s3_truncate_on_insert = 1; " 2>&1 | $CLICKHOUSE_LOCAL -q " WITH '(\\w+): (\\d+)' AS pattern, @@ -30,7 +30,7 @@ SELECT * FROM ( " echo "CHECK WITH query_log" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; SELECT type, 'S3CreateMultipartUpload', ProfileEvents['S3CreateMultipartUpload'], @@ -45,7 +45,7 @@ ORDER BY query_start_time DESC; " echo "CREATE" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS times; CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t SETTINGS @@ -56,29 +56,29 @@ CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t " echo "INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO times SELECT now() + INTERVAL 1 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "READ" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " SELECT '1', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "INSERT and READ INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO times SELECT now() + INTERVAL 2 day SETTINGS optimize_on_insert = 0; SELECT '2', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; INSERT INTO times SELECT now() + INTERVAL 3 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "DROP" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " DROP TABLE times; " echo "CHECK with query_log" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; SELECT type, query, diff --git a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh index c1f28f9f079..84be5959344 100755 --- a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh +++ b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_replicated; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_without_merge; SELECT 'With merge replicated any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_replicated; SELECT 'With merge replicated partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active; diff --git a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh index 2a94f940327..a540a0981b3 100755 --- a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh +++ b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh @@ -9,6 +9,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo " DROP USER IF EXISTS postgresql_user; CREATE USER postgresql_user HOST IP '127.0.0.1' IDENTIFIED WITH no_password; -" | $CLICKHOUSE_CLIENT -n +" | $CLICKHOUSE_CLIENT psql --host localhost --port ${CLICKHOUSE_PORT_POSTGRESQL} ${CLICKHOUSE_DATABASE} --user postgresql_user -c "SELECT 1.23::Decimal256(70) AS test;" diff --git a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh index cfb38c60615..dcd83f9fec3 100755 --- a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh +++ b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh @@ -12,7 +12,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # too slow with this. # # Unfortunately, the test has to buffer it in memory. -$CLICKHOUSE_CLIENT --max_memory_usage 16G -nm -q " +$CLICKHOUSE_CLIENT --max_memory_usage 16G -m -q " INSERT INTO FUNCTION s3('http://localhost:11111/test/$CLICKHOUSE_DATABASE/test_INT_MAX.tsv', '', '', 'TSV') SELECT repeat('a', 1024) FROM numbers((pow(2, 30) * 2) / 1024) SETTINGS s3_max_single_part_upload_size = '5Gi'; diff --git a/tests/queries/0_stateless/02706_show_columns.reference b/tests/queries/0_stateless/02706_show_columns.reference index 29e5329d63d..693a2fc2eb5 100644 --- a/tests/queries/0_stateless/02706_show_columns.reference +++ b/tests/queries/0_stateless/02706_show_columns.reference @@ -33,6 +33,7 @@ c String NO PRI SOR \N c String NO PRI SOR \N c String NO PRI SOR \N c String NO PRI SOR \N +c String NO PRI SOR \N --- Original table int32 Nullable(Int32) YES \N str String NO SOR \N diff --git a/tests/queries/0_stateless/02706_show_columns.sql b/tests/queries/0_stateless/02706_show_columns.sql index b1a907c5c71..57d80a1c580 100644 --- a/tests/queries/0_stateless/02706_show_columns.sql +++ b/tests/queries/0_stateless/02706_show_columns.sql @@ -55,6 +55,11 @@ CREATE TABLE NULL (c String) ENGINE = MergeTree ORDER BY c; SHOW COLUMNS FROM NULL; DROP TABLE NULL; +DROP TABLE IF EXISTS `tab.with.dots`; +CREATE TABLE `tab.with.dots` (c String) ENGINE = MergeTree ORDER BY c; +SHOW COLUMNS FROM `tab.with.dots`; +DROP TABLE `tab.with.dots`; + DROP DATABASE IF EXISTS `'`; CREATE DATABASE `'`; CREATE TABLE `'`.`'` (c String) ENGINE = MergeTree ORDER BY c; diff --git a/tests/queries/0_stateless/02724_show_indexes.reference b/tests/queries/0_stateless/02724_show_indexes.reference index ac0461fc506..f4308db121f 100644 --- a/tests/queries/0_stateless/02724_show_indexes.reference +++ b/tests/queries/0_stateless/02724_show_indexes.reference @@ -34,6 +34,12 @@ tbl 1 set_idx 1 \N 0 \N \N \N SET YES e --- Check with weird table names $4@^7 1 PRIMARY 1 c A 0 \N \N \N PRIMARY YES NULL 1 PRIMARY 1 c A 0 \N \N \N PRIMARY YES +tab.with.dots 1 blf_idx 1 \N 0 \N \N \N BLOOM_FILTER YES d, b +tab.with.dots 1 mm1_idx 1 \N 0 \N \N \N MINMAX YES a, c, d +tab.with.dots 1 mm2_idx 1 \N 0 \N \N \N MINMAX YES c, d, e +tab.with.dots 1 PRIMARY 1 c A 0 \N \N \N PRIMARY YES +tab.with.dots 1 PRIMARY 2 a A 0 \N \N \N PRIMARY YES +tab.with.dots 1 set_idx 1 \N 0 \N \N \N SET YES e \' 1 PRIMARY 1 c A 0 \N \N \N PRIMARY YES \' 1 PRIMARY 1 c A 0 \N \N \N PRIMARY YES --- Original table diff --git a/tests/queries/0_stateless/02724_show_indexes.sql b/tests/queries/0_stateless/02724_show_indexes.sql index 04a481fea4e..a8d699ddb47 100644 --- a/tests/queries/0_stateless/02724_show_indexes.sql +++ b/tests/queries/0_stateless/02724_show_indexes.sql @@ -43,6 +43,24 @@ CREATE TABLE NULL (c String) ENGINE = MergeTree ORDER BY c; SHOW INDEX FROM NULL; DROP TABLE NULL; +DROP TABLE IF EXISTS `tab.with.dots`; +CREATE TABLE `tab.with.dots` +( + a UInt64, + b UInt64, + c UInt64, + d UInt64, + e UInt64, + INDEX mm1_idx (a, c, d) TYPE minmax, + INDEX mm2_idx (c, d, e) TYPE minmax, + INDEX set_idx (e) TYPE set(100), + INDEX blf_idx (d, b) TYPE bloom_filter(0.8) +) +ENGINE = MergeTree +PRIMARY KEY (c, a); +SHOW INDEX FROM `tab.with.dots`; +DROP TABLE `tab.with.dots`; + DROP DATABASE IF EXISTS `'`; CREATE DATABASE `'`; CREATE TABLE `'`.`'` (c String) ENGINE = MergeTree ORDER BY c; diff --git a/tests/queries/0_stateless/02725_start_stop_fetches.sh b/tests/queries/0_stateless/02725_start_stop_fetches.sh index c9922455d94..604d0774b83 100755 --- a/tests/queries/0_stateless/02725_start_stop_fetches.sh +++ b/tests/queries/0_stateless/02725_start_stop_fetches.sh @@ -10,7 +10,7 @@ set -e NUM_REPLICAS=5 for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS r$i SYNC; CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1; " diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh index 78659b70129..6e40495157b 100755 --- a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh +++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for DISK in s3_disk s3_cache do - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " DROP TABLE IF EXISTS test; CREATE TABLE test (id Int32, empty Array(Int32)) ENGINE=MergeTree ORDER BY id @@ -17,13 +17,13 @@ do SELECT * FROM test; " - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " BACKUP TABLE test TO Disk('backups', 'test_s3_backup'); DROP TABLE test; RESTORE TABLE test FROM Disk('backups', 'test_s3_backup'); " &>/dev/null - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " SELECT * FROM test; SELECT empty FROM test; " diff --git a/tests/queries/0_stateless/02735_system_zookeeper_connection.reference b/tests/queries/0_stateless/02735_system_zookeeper_connection.reference index a2e987666e4..4eb6092137b 100644 --- a/tests/queries/0_stateless/02735_system_zookeeper_connection.reference +++ b/tests/queries/0_stateless/02735_system_zookeeper_connection.reference @@ -1,2 +1,2 @@ -default 127.0.0.1 9181 0 0 0 1 1 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS'] +default 127.0.0.1 9181 0 0 0 1 1 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE'] zookeeper2 localhost 9181 0 0 0 1 diff --git a/tests/queries/0_stateless/02766_prql.sh b/tests/queries/0_stateless/02766_prql.sh index 85b1167027c..d1e4dca070c 100755 --- a/tests/queries/0_stateless/02766_prql.sh +++ b/tests/queries/0_stateless/02766_prql.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " CREATE TEMPORARY TABLE IF NOT EXISTS aboba ( user_id UInt32, diff --git a/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh b/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh index 756488076f9..a33baaf995c 100755 --- a/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh +++ b/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh @@ -9,4 +9,4 @@ select * from numbers(1) into outfile '/dev/null'; select * from numbers(1) into outfile '/dev/null' and stdout; select * from numbers(1) into outfile '/dev/null' append; select * from numbers(1) into outfile '/dev/null' append and stdout; -" | clickhouse-format -n +" | ${CLICKHOUSE_FORMAT} -n diff --git a/tests/queries/0_stateless/02770_async_buffer_ignore.sh b/tests/queries/0_stateless/02770_async_buffer_ignore.sh index 37f002767d6..af9f18b232f 100755 --- a/tests/queries/0_stateless/02770_async_buffer_ignore.sh +++ b/tests/queries/0_stateless/02770_async_buffer_ignore.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -17,7 +17,7 @@ INSERT INTO test_s3 SELECT number, number FROM numbers(1000000); query="SELECT sum(b) FROM test_s3 WHERE a >= 100000 AND a <= 102000" query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['S3ReadRequestsCount'], ProfileEvents['ReadBufferFromS3Bytes'], diff --git a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh index c69c635f6ed..53d2832c589 100755 --- a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh +++ b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() @@ -22,7 +22,7 @@ INSERT INTO test SELECT number, randomString(100) FROM numbers(1000000); " QUERY_ID=$RANDOM -$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -nm -q " +$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -m -q " SET enable_filesystem_cache_log = 1; SYSTEM DROP FILESYSTEM CACHE; SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null; @@ -49,14 +49,14 @@ WHERE query_id = '$QUERY_ID' " # File segments cannot be less that 20Mi, # except for last file segment in a file or if file size is less. -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size AND file_segment_size < 20 * 1024 * 1024; " -all=$($CLICKHOUSE_CLIENT -nm -q " +all=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size; ") @@ -68,7 +68,7 @@ else echo "FAIL" fi -count=$($CLICKHOUSE_CLIENT -nm -q " +count=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size @@ -87,21 +87,21 @@ FROM (SELECT * FROM ($query)) AS cache_log INNER JOIN system.filesystem_cache AS cache ON cache_log.cache_path = cache.cache_path " -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND downloaded_size < 20 * 1024 * 1024; " -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB'); " -all=$($CLICKHOUSE_CLIENT -nm -q " +all=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size; ") @@ -112,7 +112,7 @@ else echo "FAIL" fi -count2=$($CLICKHOUSE_CLIENT -nm -q " +count2=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 5a37d51233d..239485ab8dd 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -25,7 +25,7 @@ do query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -nm --query " + RES=$(${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' @@ -41,7 +41,7 @@ done while true do - query_id=$(${CLICKHOUSE_CLIENT} -nq " + query_id=$(${CLICKHOUSE_CLIENT} -q " create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; set insert_keeper_fault_injection_probability=0; insert into mut values (1, 2, 3), (10, 20, 30); @@ -60,7 +60,7 @@ do ) limit 1 settings max_threads=1; " 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -nm --query " + RES=$(${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' diff --git a/tests/queries/0_stateless/02801_backup_native_copy.sh b/tests/queries/0_stateless/02801_backup_native_copy.sh index b8ee97a7c7d..bc485f903bc 100755 --- a/tests/queries/0_stateless/02801_backup_native_copy.sh +++ b/tests/queries/0_stateless/02801_backup_native_copy.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); @@ -16,28 +16,28 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_no_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " diff --git a/tests/queries/0_stateless/02803_backup_tmp_files.sh b/tests/queries/0_stateless/02803_backup_tmp_files.sh index d86beae4923..2208d3d32ef 100755 --- a/tests/queries/0_stateless/02803_backup_tmp_files.sh +++ b/tests/queries/0_stateless/02803_backup_tmp_files.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh index 63fa60bd548..4d7ab4063be 100755 --- a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh +++ b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -17,17 +17,17 @@ SETTINGS disk = disk(name = 's3_disk', type = cache, max_size = '100Ki', path = disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = ${CLICKHOUSE_TEST_UNIQUE_NAME}, disk = s3_disk); """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh index 8a4a2e906b0..a7dfa035a22 100755 --- a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh +++ b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -22,29 +22,29 @@ query_id=$RANDOM $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Null SETTINGS enable_filesystem_cache_log = 1" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY kek; """ 2>&1 | grep -q "Invalid cache key hex: kek" && echo "OK" || echo "FAIL" ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -nm --query """ +key=$($CLICKHOUSE_CLIENT -m --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -offset=$($CLICKHOUSE_CLIENT -nm --query """ +offset=$($CLICKHOUSE_CLIENT -m --query """ SELECT offset FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key OFFSET $offset; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ @@ -54,18 +54,18 @@ $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Nul ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -nm --query """ +key=$($CLICKHOUSE_CLIENT -m --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ diff --git a/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference b/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference index b91a4dd2f68..254e59d479a 100644 --- a/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference +++ b/tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.reference @@ -1,15 +1,15 @@ Prewhere filter - Prewhere filter column: and(notEmpty(v), equals(k, 3)) (removed) + Prewhere filter column: and(equals(k, 3), notEmpty(v)) (removed) 1 Prewhere filter - Prewhere filter column: and(like(d, \'%es%\'), less(c, 20), equals(b, \'3\'), equals(a, 3)) (removed) + Prewhere filter column: and(equals(a, 3), equals(b, \'3\'), less(c, 20), like(d, \'%es%\')) (removed) 1 Prewhere filter - Prewhere filter column: and(like(d, \'%es%\'), less(c, 20), greater(c, 0), equals(a, 3)) (removed) + Prewhere filter column: and(equals(a, 3), less(c, 20), greater(c, 0), like(d, \'%es%\')) (removed) 1 Prewhere filter - Prewhere filter column: and(like(d, \'%es%\'), equals(b, \'3\'), less(c, 20)) (removed) + Prewhere filter column: and(equals(b, \'3\'), less(c, 20), like(d, \'%es%\')) (removed) 1 Prewhere filter - Prewhere filter column: and(like(d, \'%es%\'), equals(b, \'3\'), equals(a, 3)) (removed) + Prewhere filter column: and(equals(a, 3), equals(b, \'3\'), like(d, \'%es%\')) (removed) 1 diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh index a2b1a953e24..ecd73bcc8a0 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS data; DROP TABLE IF EXISTS data_1; DROP TABLE IF EXISTS data_2; diff --git a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh index 16ac095312c..e6a1d547945 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple(); insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh index 4650415c202..830e78e1e9f 100755 --- a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh +++ b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, disk='s3_disk'; -- reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds @@ -15,7 +15,7 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup2') SETTINGS allow_s3_native_copy=1" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT 'native_copy', @@ -26,7 +26,7 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup3') SETTINGS allow_s3_native_copy=0" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT 'no_native_copy', diff --git a/tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.reference b/tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.reference index eb5e685597c..c4ef127ebc0 100644 --- a/tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.reference +++ b/tests/queries/0_stateless/02864_statistics_delayed_materialization_in_merge.reference @@ -5,8 +5,8 @@ After insert After merge Prewhere info Prewhere filter - Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed) + Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed) After truncate, insert, and materialize Prewhere info Prewhere filter - Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed) + Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed) diff --git a/tests/queries/0_stateless/02864_statistics_usage.reference b/tests/queries/0_stateless/02864_statistics_usage.reference index a9f669b88c1..fd4181a59c3 100644 --- a/tests/queries/0_stateless/02864_statistics_usage.reference +++ b/tests/queries/0_stateless/02864_statistics_usage.reference @@ -1,7 +1,7 @@ After insert Prewhere info Prewhere filter - Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed) + Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed) After drop statistic Prewhere info Prewhere filter @@ -9,12 +9,12 @@ After drop statistic After add and materialize statistic Prewhere info Prewhere filter - Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed) + Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed) After merge Prewhere info Prewhere filter - Prewhere filter column: and(less(a, 10_UInt8), less(b, 10_UInt8)) (removed) + Prewhere filter column: and(less(b, 10_UInt8), less(a, 10_UInt8)) (removed) After rename Prewhere info Prewhere filter - Prewhere filter column: and(less(a, 10_UInt8), less(c, 10_UInt8)) (removed) + Prewhere filter column: and(less(c, 10_UInt8), less(a, 10_UInt8)) (removed) diff --git a/tests/queries/0_stateless/02867_storage_set_tsan.sh b/tests/queries/0_stateless/02867_storage_set_tsan.sh index 81ae5f0bda8..f6b684c0a55 100755 --- a/tests/queries/0_stateless/02867_storage_set_tsan.sh +++ b/tests/queries/0_stateless/02867_storage_set_tsan.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -mn -q """ +$CLICKHOUSE_CLIENT -m -q """ DROP TABLE IF EXISTS t1_02867; CREATE TABLE t1_02867 (x UInt64) ENGINE=Set(); """ @@ -39,4 +39,4 @@ repeat_truncate_insert & sleep 10 -$CLICKHOUSE_CLIENT -mn -q "DROP TABLE IF EXISTS t1_02867;" +$CLICKHOUSE_CLIENT -m -q "DROP TABLE IF EXISTS t1_02867;" diff --git a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh index a0fdb5276e0..a091645d1e3 100755 --- a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh +++ b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh @@ -10,14 +10,14 @@ echo '{"a" : 1, "obj" : {"f1" : 1, "f2" : "2020-01-01"}}' > $CLICKHOUSE_TEST_UNI echo '{"b" : 2, "obj" : {"f3" : 2, "f2" : "Some string"}}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data2.jsonl echo '{"c" : "hello"}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); @@ -25,14 +25,14 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); cd $CLICKHOUSE_TEST_UNIQUE_NAME/ && tar -cf archive.tar data1.jsonl data2.jsonl data3.jsonl && cd .. -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); @@ -41,7 +41,7 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); echo 'Error' > $CLICKHOUSE_TEST_UNIQUE_NAME/data4.jsonl $CLICKHOUSE_LOCAL -q "desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl') settings schema_inference_mode='union'" 2>&1 | grep -c -F "CANNOT_EXTRACT_TABLE_STRUCTURE" -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{2,3}.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl'); diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 81ba59fc591..91eeb22c19e 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -67,7 +67,7 @@ curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(is_leader wait; -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; -- Check that number of ZK request is less then a half of (total replicas * concurrency) diff --git a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh index d1cbc54d294..2eccade5c81 100755 --- a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh +++ b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh @@ -8,7 +8,7 @@ CURDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists r1; drop table if exists r2; @@ -64,7 +64,7 @@ function insert_duplicates() { wait - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " system sync replica r1; system sync replica r2; " @@ -84,7 +84,7 @@ function loop() do while ! insert_duplicates do - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " truncate table r1; truncate table r2; system sync replica r1; @@ -137,8 +137,8 @@ function list_keeper_nodes() { list_keeper_nodes "${table_shared_id}" -$CLICKHOUSE_CLIENT -nm -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & -$CLICKHOUSE_CLIENT -nm -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -m -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -m -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & wait list_keeper_nodes "${table_shared_id}" diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh index 2b92a113e91..057f76e63d0 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh @@ -10,11 +10,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Basic refreshing. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view a refresh after 2 second engine Memory @@ -23,41 +23,41 @@ $CLICKHOUSE_CLIENT -nq " select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes; show create a;" # Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] do sleep 0.5 done -start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" +start_time="`$CLICKHOUSE_CLIENT -q "select reinterpret(now64(), 'Int64')"`" # Check table contents. -$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" +$CLICKHOUSE_CLIENT -q "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" # Wait for table contents to change. -res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" +res1="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values'`" while : do - res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + res2="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" [ "$res2" == "$res1" ] || break sleep 0.5 done # Wait for another change. while : do - res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + res3="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" [ "$res3" == "$res2" ] || break sleep 0.5 done # Check that the two changes were at least 1 second apart, in particular that we're not refreshing # like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer # to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); select '<4: next refresh in>', next_refresh_time-last_refresh_time from refreshes;" # Create a source table from which views will read. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table src (x Int8) engine Memory as select 1;" # Switch to fake clock, change refresh schedule, change query. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view a set fake time '2050-01-01 00:00:01'; system wait view a; system refresh view a; @@ -68,19 +68,19 @@ $CLICKHOUSE_CLIENT -nq " select '<4.5: altered>', status, last_refresh_result, next_refresh_time from refreshes; show create a;" # Advance time to trigger the refresh. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<5: no refresh>', count() from a; system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<6: refreshed>', * from a; select '<7: refreshed>', status, last_refresh_result, next_refresh_time from refreshes;" # Create a dependent view, refresh it once. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; show create b; system test view b set fake time '2052-11-11 11:11:11'; @@ -88,89 +88,89 @@ $CLICKHOUSE_CLIENT -nq " system wait view b; select '<7.5: created dependent>', last_refresh_time from refreshes where view = 'b';" # Next refresh shouldn't start until the dependency refreshes. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<8: refreshed>', * from b; select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes; system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] do sleep 0.5 done # Drop the source table, check that refresh fails and doesn't leave a temp table behind. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); drop table src; system refresh view a;" -$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -q " select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" # Create the source table again, check that refresh succeeds (in particular that tables are looked # up by name rather than uuid). -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<10: creating>', view, status, remaining_dependencies, next_refresh_time from refreshes; create table src (x Int16) engine Memory as select 2; system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Both tables should've refreshed. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<11: chain-refreshed a>', * from a; select '<12: chain-refreshed b>', * from b; select '<13: chain-refreshed>', view, status, remaining_dependencies, last_refresh_result, last_refresh_time, next_refresh_time, exception == '' from refreshes;" # Make the dependent table run ahead by one refresh cycle, make sure it waits for the dependency to # catch up to the same cycle. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2059-01-01 00:00:00'; system refresh view b;" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2061-01-01 00:00:00'; system test view a set fake time '2057-01-01 00:00:00';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<14: waiting for next cycle>', view, status, remaining_dependencies, next_refresh_time from refreshes; truncate src; insert into src values (3); system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<15: chain-refreshed a>', * from a; select '<16: chain-refreshed b>', * from b; select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" # Get to WaitingForDependencies state and remove the depencency. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<18: removed dependency>', view, status, remaining_dependencies, last_refresh_time,next_refresh_time, refresh_count from refreshes where view = 'b'; show create b;" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table src; drop table a; drop table b; diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh index 50a905576d5..2d00d61f253 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh @@ -12,29 +12,29 @@ CLICKHOUSE_LOG_COMMENT= CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --allow_materialized_view_with_bad_select=0 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Select from a table that doesn't exist, get an exception. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table src (x Int8) engine Memory as select 1; create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; drop table src;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] do sleep 0.5 done # Check exception, create src, expect successful refresh. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' ? '1' : exception from refreshes where view = 'c'; create table src (x Int64) engine Memory as select 1; system refresh view c;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Rename table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<20: unexception>', * from c; rename table c to d; select '<21: rename>', * from d; @@ -42,130 +42,130 @@ $CLICKHOUSE_CLIENT -nq " # Do various things during a refresh. # First make a nonempty view. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table d; truncate src; insert into src values (1); create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x empty as select x + sleepEachRow(1) as x from src settings max_block_size = 1;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Stop refreshes. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<23: simple refresh>', * from e; system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] do sleep 0.5 done # Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure # we wait for a slow refresh, not a previous fast one.) -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " insert into src select * from numbers(1000) settings max_block_size=1; system start view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Rename. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " rename table e to f; select '<24: rename during refresh>', * from f; select '<25: rename during refresh>', view, status from refreshes where view = 'f'; alter table f modify refresh after 10 year;" # Cancel. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Check that another refresh doesn't immediately start after the cancelled one. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<27: cancelled>', view, status, last_refresh_result from refreshes where view = 'f'; system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Drop. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table f; select '<28: drop during refresh>', view, status from refreshes; select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" # Try OFFSET and RANDOMIZE FOR. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; show create g; system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " with '2050-02-10 04:00:00'::DateTime as expected select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" # Send data 'TO' an existing table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table g; create table dest (x Int64) engine MergeTree order by x; truncate src; insert into src values (1); create materialized view h refresh every 1 second to dest empty as select x*10 as x from src; show create h;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<30: to existing table>', * from dest; insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] +while [ "`$CLICKHOUSE_CLIENT -q "select count() from dest -- $LINENO" | xargs`" != '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<31: to existing table>', * from dest; drop table dest; drop table h;" # Retries. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" -$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -q " select '<31.5: will retry>', last_refresh_result, retry > 0 from refreshes; create table src2 (x Int8) engine Memory; insert into src2 values (1); exchange tables src and src2; drop table src2;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<31.6: did retry>', x from h2; drop table h2" # EMPTY -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] +while [ "`$CLICKHOUSE_CLIENT -q "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<32: empty>', view, status, last_refresh_result, retry from refreshes order by view; drop table i; drop table j;" # APPEND -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; select '<33: append>', * from k; system refresh view k; @@ -177,7 +177,7 @@ $CLICKHOUSE_CLIENT -nq " system wait view k; select '<35: append>', * from k order by x;" # ALTER to non-APPEND -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " alter table k modify refresh every 10 year; system wait view k; system refresh view k; @@ -187,7 +187,7 @@ $CLICKHOUSE_CLIENT -nq " truncate table src;" # APPEND + TO + regular materialized view reading from it. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table mid (x Int64) engine MergeTree order by x; create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; create materialized view m (x Int64) engine Memory as select x*10 as x from mid; @@ -204,19 +204,19 @@ $CLICKHOUSE_CLIENT -nq " drop table mid;" # Failing to create inner table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); drop table n;" # Reading from table that doesn't exist yet. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nope.nonexist; -- { serverError UNKNOWN_DATABASE } create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nope.nonexist settings allow_materialized_view_with_bad_select = 1; drop table o;" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table refreshes;" diff --git a/tests/queries/0_stateless/02941_variant_type_1.sh b/tests/queries/0_stateless/02941_variant_type_1.sh index c12bece6d54..ef1d1deff4e 100755 --- a/tests/queries/0_stateless/02941_variant_type_1.sh +++ b/tests/queries/0_stateless/02941_variant_type_1.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test1_insert() { echo "test1 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number from numbers(3); insert into test select number + 6, ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(3); @@ -21,7 +21,7 @@ insert into test select number + 15, range(number + 1)::Array(UInt64) from numbe function test1_select() { echo "test1 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -36,7 +36,7 @@ select v.\`Array(UInt64)\`.size0 from test order by id;" function test2_insert() { echo "test2 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number % 2 ? NULL : number from numbers(3); insert into test select number + 6, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(('lc_str_' || toString(number))::LowCardinality(String), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3); @@ -47,7 +47,7 @@ insert into test select number + 15, number % 2 ? CAST(NULL, 'Variant(String, UI function test2_select() { echo "test2 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -68,7 +68,7 @@ function test3_insert() function test3_select() { echo "test3 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; diff --git a/tests/queries/0_stateless/02941_variant_type_2.sh b/tests/queries/0_stateless/02941_variant_type_2.sh index e93dfac8510..7017edac525 100755 --- a/tests/queries/0_stateless/02941_variant_type_2.sh +++ b/tests/queries/0_stateless/02941_variant_type_2.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test4_insert() { echo "test4 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(100000); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(100000); insert into test select number + 100000, number from numbers(100000); insert into test select number + 200000, ('str_' || toString(number))::Variant(String) from numbers(100000); insert into test select number + 300000, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(100000); @@ -21,7 +21,7 @@ insert into test select number + 500000, range(number % 20 + 1)::Array(UInt64) f function test4_select { echo "test4 select" - $CH_CLIENT -nmq "select v from test format Null; + $CH_CLIENT -mq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02941_variant_type_3.sh b/tests/queries/0_stateless/02941_variant_type_3.sh index cc0fde5b689..5d923a47e9b 100755 --- a/tests/queries/0_stateless/02941_variant_type_3.sh +++ b/tests/queries/0_stateless/02941_variant_type_3.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test5_insert() { echo "test5 insert" - $CH_CLIENT -nmq " + $CH_CLIENT -mq " insert into test select number, NULL from numbers(200000); insert into test select number + 200000, number % 2 ? NULL : number from numbers(200000); insert into test select number + 400000, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(200000); @@ -22,7 +22,7 @@ insert into test select number + 1000000, number % 2 ? CAST(NULL, 'Variant(Strin function test5_select() { echo "test5 select" - $CH_CLIENT -nmq " + $CH_CLIENT -mq " select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; diff --git a/tests/queries/0_stateless/02941_variant_type_4.sh b/tests/queries/0_stateless/02941_variant_type_4.sh index 93a1770d05e..88f6a475d46 100755 --- a/tests/queries/0_stateless/02941_variant_type_4.sh +++ b/tests/queries/0_stateless/02941_variant_type_4.sh @@ -17,7 +17,7 @@ function test6_insert() function test6_select() { echo "test6 select" - $CH_CLIENT -nmq "select v from test format Null; + $CH_CLIENT -mq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh index cb099bb59ae..f9e84dc2b50 100755 --- a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh +++ b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh @@ -10,7 +10,7 @@ disk_name="s3_cache_02944" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(100); @@ -33,7 +33,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -47,7 +47,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -63,7 +63,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -77,7 +77,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" diff --git a/tests/queries/0_stateless/02960_polygon_bound_bug.sh b/tests/queries/0_stateless/02960_polygon_bound_bug.sh index 0c3db01a77c..1e9be36da55 100755 --- a/tests/queries/0_stateless/02960_polygon_bound_bug.sh +++ b/tests/queries/0_stateless/02960_polygon_bound_bug.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_LOCAL -nm -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; +$CLICKHOUSE_LOCAL -m -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; INSERT INTO test_table SELECT * FROM file('$CURDIR/data_parquet/02960_polygon_bound_bug.parquet', Parquet); CREATE DICTIONARY test_dict (geom MultiPolygon) PRIMARY KEY geom SOURCE (CLICKHOUSE(TABLE 'test_table')) LIFETIME(MIN 0 MAX 0) LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1)); SELECT dictHas(test_dict,(174.84729269276494,-36.99524960275426));" diff --git a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh index 145b921a750..1bd86ee0c75 100755 --- a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh +++ b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh @@ -24,7 +24,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must be unique across the policy' @@ -40,7 +40,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must cover the range from 1 to N (lowest priority specified) without gaps' diff --git a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh index 24fe964b824..3b57941452d 100755 --- a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh +++ b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02963_remote_read_bug" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, s String) @@ -22,7 +22,7 @@ OPTIMIZE TABLE test FINAL; query_id=$(random_str 10) -$CLICKHOUSE_CLIENT -nm --query_id "$query_id" --query " +$CLICKHOUSE_CLIENT -m --query_id "$query_id" --query " WITH RANDOM_SET AS ( SELECT rand32() % 10000 FROM numbers(100) ) @@ -37,7 +37,7 @@ SETTINGS merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem = 1, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem = 1; " -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " SYSTEM FLUSH LOGS; -- This threshold was determined experimentally - before the fix this ratio had values around 50K diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh index 1089eb4051f..b4271c3d29b 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh index 7a0e2d9bfdb..ed13bf3321b 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh index a595e363ef4..4be4ab81c68 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02969_auto_format_detection.sh b/tests/queries/0_stateless/02969_auto_format_detection.sh index 88d6575e499..ab21dbb3df8 100755 --- a/tests/queries/0_stateless/02969_auto_format_detection.sh +++ b/tests/queries/0_stateless/02969_auto_format_detection.sh @@ -24,12 +24,12 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE', auto, 'a UInt64, b String, c Array(UInt64), d Tuple(a UInt64, b String)')" -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE'); desc file('$DATA_FILE'); " -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE', JSONEachRow); desc file('$DATA_FILE'); " @@ -39,7 +39,7 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}')" $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}') settings schema_inference_mode='union'" 2>&1 | grep -c "CANNOT_DETECT_FORMAT" -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE.2'); desc file('$DATA_FILE.{1,2}'); " diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh index d543f7195a9..a46f0018ad3 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by key; insert into data values (1); @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -nm -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table data; attach table data (key Int) engine=MergeTree() order by key settings diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh index eec05c81344..522fdd6096a 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data_read; drop table if exists data_write; @@ -33,7 +33,7 @@ $CLICKHOUSE_CLIENT -nm -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data_read to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table data_read; attach table data_read (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', 'read') order by key settings @@ -57,7 +57,7 @@ echo "Files before DETACH TABLE" # sed to match any part, since in case of fault injection part name may not be all_0_0_0 but all_1_1_0 clickhouse-disks -C "$config" --disk s3_plain_disk --query "list --recursive $path" | tail -n+2 | sed 's/all_[^_]*_[^_]*_0/all_X_X_X/g' -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " detach table data_read; detach table data_write; " diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh index ef4866cb69e..fca8b7a5e97 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh index dd4b1f7cec0..08d5ae3a6b2 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh index 33386c76edb..678b1db4d68 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh index b66ef83abf2..606fda6d4fd 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh index f9e1838f491..b571631dc8f 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh index 698e70d4064..e4358011e50 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh index 8eea7940774..c13674b9bd1 100755 --- a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh +++ b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query "drop table if exists test_s3_mt" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " create table test_s3_mt (a Int32, b Int64, c Int64) engine = MergeTree() partition by intDiv(a, 1000) order by tuple(a, b) settings disk = disk( name = 03008_s3_plain_rewritable, @@ -19,7 +19,7 @@ settings disk = disk( secret_access_key = clickhouse); " -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " insert into test_s3_mt (*) values (1, 2, 0), (2, 2, 2), (3, 1, 9), (4, 7, 7), (5, 10, 2), (6, 12, 5); insert into test_s3_mt (*) select number, number, number from numbers_mt(10000); select count(*) from test_s3_mt; @@ -31,13 +31,13 @@ ${CLICKHOUSE_CLIENT} --query "optimize table test_s3_mt final" ${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt add projection test_s3_mt_projection (select * order by b)" 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt update c = 0 where a % 2 = 1; alter table test_s3_mt add column d Int64 after c; alter table test_s3_mt drop column c; " 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " detach table test_s3_mt; attach table test_s3_mt; " diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh index cba5317fcfa..278ad6f0654 100755 --- a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh +++ b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="s3_cache" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(1000); @@ -26,7 +26,7 @@ sed -i "s|$prev_max_size<\/max_size>|$new_max_size<\/max_siz # echo $prev_max_size # echo $new_max_size -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" @@ -36,7 +36,7 @@ $CLICKHOUSE_CLIENT --query "SELECT current_size <= max_size FROM system.filesyst sed -i "s|$new_max_size<\/max_size>|$prev_max_size<\/max_size>|" $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" diff --git a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh index 9ea86105a3a..8183b636549 100755 --- a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh index 9a2a6dd957c..ddafa9db210 100755 --- a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh @@ -16,7 +16,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh index 0199035a3df..b1046ca08ba 100755 --- a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh index e2ea5bc3466..f3693a2e7b1 100755 --- a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh index 43607cf95a8..bba92a2ee06 100755 --- a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, number >= 75000 ? 2 : 1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03151_unload_index_race.sh b/tests/queries/0_stateless/03151_unload_index_race.sh index 7e9dfa7cddc..4cdf06abae2 100755 --- a/tests/queries/0_stateless/03151_unload_index_race.sh +++ b/tests/queries/0_stateless/03151_unload_index_race.sh @@ -42,8 +42,8 @@ function thread_alter_settings() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" - $CLICKHOUSE_CLIENT -n --query "SYSTEM UNLOAD PRIMARY KEY t" + $CLICKHOUSE_CLIENT --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" + $CLICKHOUSE_CLIENT --query "SYSTEM UNLOAD PRIMARY KEY t" sleep 0.0$RANDOM done } @@ -52,7 +52,7 @@ function thread_query_table() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - COUNT=$($CLICKHOUSE_CLIENT -n --query "SELECT count() FROM t where not ignore(*);") + COUNT=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM t where not ignore(*);") if [ "$COUNT" -ne "2000" ]; then echo "$COUNT" fi diff --git a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh index 22a2fd82c64..d1afafa77a5 100755 --- a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh +++ b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh @@ -15,7 +15,7 @@ errors_222=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHER errors_333=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHERE code = 333") # Throw three random errors: 111, 222 and 333 and wait for more than collect_interval_milliseconds to ensure system.error_log is flushed -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -24,14 +24,14 @@ SYSTEM FLUSH LOGS; " # Check that the three random errors are propagated -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT sum(value) > $errors_111 FROM system.error_log WHERE code = 111; SELECT sum(value) > $errors_222 FROM system.error_log WHERE code = 222; SELECT sum(value) > $errors_333 FROM system.error_log WHERE code = 333; " # Ensure that if we throw them again, they're still propagated -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -39,7 +39,7 @@ SELECT sleep(2) format NULL; SYSTEM FLUSH LOGS; " -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT sum(value) > $(($errors_111+1)) FROM system.error_log WHERE code = 111; SELECT sum(value) > $(($errors_222+1)) FROM system.error_log WHERE code = 222; SELECT sum(value) > $(($errors_333+1)) FROM system.error_log WHERE code = 333; diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference new file mode 100644 index 00000000000..297ef667995 --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -0,0 +1,66 @@ +localhost 9000 0 0 0 +localhost 9000 0 0 0 +Basic authentication after user creation +1 +localhost 9000 0 0 0 +Changed password, old password should not work +AUTHENTICATION_FAILED +New password should work +1 +localhost 9000 0 0 0 +Two new passwords were added, should both work +1 +1 +localhost 9000 0 0 0 +Authenticating with ssh key +1 +Altering credentials and keeping only bcrypt_password +localhost 9000 0 0 0 +Asserting SSH does not work anymore +AUTHENTICATION_FAILED +Asserting bcrypt_password works +1 +Adding new bcrypt_password +localhost 9000 0 0 0 +Both current authentication methods should work +1 +1 +Reset authentication methods to new +localhost 9000 0 0 0 +Only the latest should work, below should fail +AUTHENTICATION_FAILED +Should work +1 +Multiple identified with, not allowed +Syntax error +localhost 9000 0 0 0 +CREATE Multiple identified with, not allowed +Syntax error +localhost 9000 0 0 0 +Create user with no identification +localhost 9000 0 0 0 +Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types +BAD_ARGUMENTS +Try to add no_password mixed with other authentication methods, should not be allowed +SYNTAX_ERROR +Adding no_password, should fail +SYNTAX_ERROR +Replacing existing authentication methods in favor of no_password, should succeed +localhost 9000 0 0 0 +Trying to auth with no pwd, should succeed +1 +localhost 9000 0 0 0 +Use WITH without providing authentication type, should fail +Syntax error +Create user with ADD identification, should fail, add is not allowed for create query +SYNTAX_ERROR +Trailing comma should result in syntax error +SYNTAX_ERROR +First auth method can't specify type if WITH keyword is not present +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement +SYNTAX_ERROR +ADD NOT IDENTIFIED should result in syntax error +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh new file mode 100755 index 00000000000..552e5cc20eb --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-replicated-database: https://s3.amazonaws.com/clickhouse-test-reports/65277/43e9a7ba4bbf7f20145531b384a31304895b55bc/stateless_tests__release__old_analyzer__s3__databasereplicated__[1_2].html and https://github.com/ClickHouse/ClickHouse/blob/011c694117845500c82f9563c65930429979982f/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh#L4 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +ssh_key="-----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGAAAAJApc2djKXNn + YwAAAAtzc2gtZWQyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGA + AAAEAk15S5L7j85LvmAivo2J8lo44OR/tLILBO1Wb2//mFwBzqa3duS0ce6QYkzUgko9W0 + Ux7i7d3xPoseFrwnhY4YAAAADWFydGh1ckBhcnRodXI= + -----END OPENSSH PRIVATE KEY-----" + +function test_login_no_pwd +{ + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=$1" -d "select 1" +} + +function test_login_pwd +{ + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=$1&password=$2" -d "select 1" +} + +function test_login_pwd_expect_error +{ + test_login_pwd "$1" "$2" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' | head -n 1 +} + +function test +{ + user="u01_03174$RANDOM" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password BY '1'" + + echo "Basic authentication after user creation" + test_login_pwd ${user} '1' + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password BY '2'" + + echo "Changed password, old password should not work" + test_login_pwd_expect_error ${user} '1' + + echo "New password should work" + test_login_pwd ${user} '2' + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password BY '3', plaintext_password BY '4'" + + echo "Two new passwords were added, should both work" + test_login_pwd ${user} '3' + + test_login_pwd ${user} '4' + + ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" + + echo ${ssh_key} > ssh_key + + echo "Authenticating with ssh key" + ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" + + echo "Altering credentials and keeping only bcrypt_password" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH bcrypt_password BY '5'" + + echo "Asserting SSH does not work anymore" + ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' | head -n 1 + + echo "Asserting bcrypt_password works" + test_login_pwd ${user} '5' + + echo "Adding new bcrypt_password" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH bcrypt_password BY '6'" + + echo "Both current authentication methods should work" + test_login_pwd ${user} '5' + test_login_pwd ${user} '6' + + echo "Reset authentication methods to new" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" + + echo "Only the latest should work, below should fail" + test_login_pwd_expect_error ${user} '5' + + echo "Should work" + test_login_pwd ${user} '6' + + echo "Multiple identified with, not allowed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER ${user} $1" + + echo "CREATE Multiple identified with, not allowed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + echo "Create user with no identification" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1" + + echo "Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" | head -n 1 + + echo "Try to add no_password mixed with other authentication methods, should not be allowed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "Adding no_password, should fail" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "Replacing existing authentication methods in favor of no_password, should succeed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH no_password" + + echo "Trying to auth with no pwd, should succeed" + test_login_no_pwd ${user} + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + echo "Use WITH without providing authentication type, should fail" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 + + echo "Create user with ADD identification, should fail, add is not allowed for create query" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "Trailing comma should result in syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "First auth method can't specify type if WITH keyword is not present" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "ADD NOT IDENTIFIED should result in syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + echo "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user}" + +} + +test "ON CLUSTER test_shard_localhost" diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference new file mode 100644 index 00000000000..f5da15ccfaa --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference @@ -0,0 +1,2 @@ +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql new file mode 100644 index 00000000000..f9e9e72e5c0 --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql @@ -0,0 +1,13 @@ +-- Tags: no-fasttest, no-parallel + +-- Create user with mix both implicit and explicit auth type, starting with with +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_multiple_auth_show_create; + +DROP USER IF EXISTS u_03174_multiple_auth_show_create; + +-- Create user with mix both implicit and explicit auth type, starting with by +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_multiple_auth_show_create; + +DROP USER IF EXISTS u_03174_multiple_auth_show_create; diff --git a/tests/queries/0_stateless/03176_check_timeout_in_index_analysis.reference b/tests/queries/0_stateless/03176_check_timeout_in_index_analysis.reference new file mode 100644 index 00000000000..05a83c81dae --- /dev/null +++ b/tests/queries/0_stateless/03176_check_timeout_in_index_analysis.reference @@ -0,0 +1,4 @@ +5 +03176_q1 1 0 0 +03176_q2 1 2 0 +03176_q3 0 1 1 diff --git a/tests/queries/0_stateless/03176_check_timeout_in_index_analysis.sql b/tests/queries/0_stateless/03176_check_timeout_in_index_analysis.sql new file mode 100644 index 00000000000..4163ad58c4e --- /dev/null +++ b/tests/queries/0_stateless/03176_check_timeout_in_index_analysis.sql @@ -0,0 +1,32 @@ +-- Tags: no-parallel, no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest +-- no-parallel because the test uses failpoint + +CREATE TABLE t_03176(k UInt64, v UInt64) ENGINE=MergeTree() ORDER BY k PARTITION BY k; + +INSERT INTO t_03176 SELECT number, number FROM numbers(5); + +-- Table is partitioned by k to so it will have 5 partitions +SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_03176' AND active; + +-- This query is fast without failpoint: should take < 1 sec +EXPLAIN indexes = 1 SELECT * FROM t_03176 ORDER BY k LIMIT 5 SETTINGS log_comment = '03176_q1' FORMAT Null; + +SYSTEM ENABLE FAILPOINT slowdown_index_analysis; + +-- Check that failpont actually works: the query should take >= 5 sec +EXPLAIN indexes = 1 SELECT * FROM t_03176 ORDER BY k LIMIT 5 SETTINGS log_comment = '03176_q2' FORMAT Null; + +-- Now the query should be cancelled after about 1 sec +EXPLAIN indexes = 1 SELECT * FROM t_03176 ORDER BY k LIMIT 5 SETTINGS log_comment = '03176_q3', max_execution_time = 1.1 FORMAT Null; -- { serverError TIMEOUT_EXCEEDED } + +SYSTEM DISABLE FAILPOINT slowdown_index_analysis; + +SYSTEM FLUSH LOGS; + +-- Check that q1 was fast, q2 was slow and q3 had timeout +SELECT log_comment, type = 'QueryFinish', intDiv(query_duration_ms, 2000), length(exception) > 0 +FROM system.query_log +WHERE current_database = currentDatabase() AND log_comment LIKE '03176_q_' AND type IN ('QueryFinish', 'ExceptionBeforeStart') +ORDER BY log_comment; + +DROP TABLE t_03176; diff --git a/tests/queries/0_stateless/03203_hive_style_partitioning.sh b/tests/queries/0_stateless/03203_hive_style_partitioning.sh index 60e8a6e9faa..fdd0fcd8ec5 100755 --- a/tests/queries/0_stateless/03203_hive_style_partitioning.sh +++ b/tests/queries/0_stateless/03203_hive_style_partitioning.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE FILE HIVE PARTITIONING'" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -22,20 +22,20 @@ SELECT toTypeName(array), toTypeName(float) FROM file('$CURDIR/data_hive/partiti SELECT count(*) FROM file('$CURDIR/data_hive/partitioning/number=42/date=2020-01-01/sample.parquet') WHERE number = 42; """ -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT identifier FROM file('$CURDIR/data_hive/partitioning/identifier=*/email.csv') LIMIT 2; SELECT a FROM file('$CURDIR/data_hive/partitioning/a=b/a=b/sample.parquet') LIMIT 1; """ -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/column0=Elizabeth1/sample.parquet') LIMIT 10; """ 2>&1 | grep -c "INCORRECT_DATA" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 0; SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10; @@ -45,14 +45,14 @@ SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_exis $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE URL PARTITIONING'" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; SELECT *, non_existing_column FROM url('http://localhost:11111/test/hive_partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10;""" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -62,7 +62,7 @@ SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/colum $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3 PARTITIONING'" -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -71,7 +71,7 @@ SELECT *, non_existing_column FROM s3('http://localhost:11111/test/hive_partitio SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=*/sample.parquet') WHERE column0 = 'Elizabeth' LIMIT 10; """ -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -79,7 +79,7 @@ SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3CLUSTER PARTITIONING'" -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3Cluster(test_cluster_one_shard_three_replicas_localhost, 'http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; diff --git a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh index e0c8f08e695..6bfe82a91a6 100755 --- a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh +++ b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # In this test we restore from "/tests/queries/0_stateless/backups/mt_250_parts.zip" backup_name="$($CURDIR/helpers/install_predefined_backup.sh mt_250_parts.zip)" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS manyparts; CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merge_tree_clear_old_temporary_directories_interval_seconds=1, temporary_directories_lifetime=1; " @@ -16,7 +16,7 @@ CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merg # RESTORE must protect its temporary directories from removing. ${CLICKHOUSE_CLIENT} --query "RESTORE TABLE default.mt_250_parts AS manyparts FROM Disk('backups', '${backup_name}') SETTINGS allow_different_table_def=true" | grep -o "RESTORED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " SELECT count(), sum(x) FROM manyparts; DROP TABLE manyparts; " diff --git a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference index c20baa0d261..90120f85a71 100644 --- a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference +++ b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference @@ -1,2 +1,5 @@ 1970-01-01 00:00:01.000 1970-01-01 00:00:01.000 +1970-01-01 00:00:01.000 0 +1970-01-01 00:00:01.000 0 +1970-01-01 00:00:01.000 0 localhost diff --git a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql index 63334a511c7..d01bb8d72f1 100644 --- a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql +++ b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql @@ -4,3 +4,22 @@ SET session_timezone = 'UTC'; SELECT toDateTime64('1970-01-01 00:00:01', 3) FROM remote('127.0.0.{1,2}', system, one) ; + +SELECT toDateTime64('1970-01-01 00:00:01', 3), dummy +FROM remote('127.0.0.{1,2}', system, one) +GROUP BY dummy +ORDER BY dummy +; + +SELECT materialize(toDateTime64('1970-01-01 00:00:01', 3)), dummy +FROM remote('127.0.0.{1,2}', system, one) +GROUP BY dummy +ORDER BY dummy +; + + +SELECT toDateTime64('1970-01-01 00:00:01', 3), sum(dummy), hostname() +FROM remote('127.0.0.{1,2}', system, one) +GROUP BY hostname() +ORDER BY ALL +; diff --git a/tests/queries/0_stateless/03222_json_empty_as_default.reference b/tests/queries/0_stateless/03222_json_empty_as_default.reference new file mode 100644 index 00000000000..1a98f45577a --- /dev/null +++ b/tests/queries/0_stateless/03222_json_empty_as_default.reference @@ -0,0 +1,47 @@ +-- Simple types +-- { echoOn } +SELECT x FROM format(JSONEachRow, 'x Date', '{"x":""}'); +1970-01-01 +SELECT x FROM format(JSONEachRow, 'x Date32', '{"x":""}'); +1970-01-01 +SELECT toTimeZone(x, 'UTC') FROM format(JSONEachRow, 'x DateTime', '{"x":""}'); +1970-01-01 00:00:00 +SELECT toTimeZone(x, 'UTC') FROM format(JSONEachRow, 'x DateTime64', '{"x":""}'); +1970-01-01 00:00:00.000 +SELECT x FROM format(JSONEachRow, 'x IPv4', '{"x":""}'); +0.0.0.0 +SELECT x FROM format(JSONEachRow, 'x IPv6', '{"x":""}'); +:: +SELECT x FROM format(JSONEachRow, 'x UUID', '{"x":""}'); +00000000-0000-0000-0000-000000000000 +-- { echoOn } +SELECT COUNT(DISTINCT col) FROM table1; +1 +-- { echoOn } +SELECT * FROM table1 ORDER BY address ASC; +:: +2001:db8:3333:4444:5555:6666:7777:8888 +-- Nullable +-- { echoOn } +SELECT x FROM format(JSONEachRow, 'x Nullable(IPv6)', '{"x":""}'); +\N +-- Compound types +SELECT x FROM format(JSONEachRow, 'x Array(UUID)', '{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e",""]}'); +['00000000-0000-0000-0000-000000000000','b15f852c-c41a-4fd6-9247-1929c841715e','00000000-0000-0000-0000-000000000000'] +SELECT x FROM format(JSONEachRow, 'x Array(Nullable(IPv6))', '{"x":["",""]}'); +[NULL,NULL] +SELECT x FROM format(JSONEachRow, 'x Tuple(Date, IPv4, String)', '{"x":["", "", "abc"]}'); +('1970-01-01','0.0.0.0','abc') +SELECT x FROM format(JSONEachRow, 'x Map(String, IPv6)', '{"x":{"abc": ""}}'); +{'abc':'::'} +SELECT x FROM format(JSONEachRow, 'x Variant(Date, UUID)', '{"x":""}'); +\N +-- Deep composition +SELECT x FROM format(JSONEachRow, 'x Array(Array(IPv6))', '{"x":[["2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF", ""], ["", "2001:db8:3333:4444:5555:6666:7777:8888"]]}'); +[['2001:db8:3333:4444:cccc:dddd:eeee:ffff','::'],['::','2001:db8:3333:4444:5555:6666:7777:8888']] +SELECT x FROM format(JSONEachRow, 'x Variant(Date, Array(UUID))', '{"x":["", "b15f852c-c41a-4fd6-9247-1929c841715e"]}'); +['00000000-0000-0000-0000-000000000000','b15f852c-c41a-4fd6-9247-1929c841715e'] +SELECT x FROM format(JSONEachRow, 'x Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))', '{"x":[[""], ["",{"abc":""}]]}'); +(['00000000-0000-0000-0000-000000000000'],('00000000-0000-0000-0000-000000000000',{'abc':'::'})) +SELECT x FROM format(JSONEachRow, 'x Map(Tuple(Date,IPv4), Variant(UUID,IPv6))', '{"x":{["",""]:""}}'); +{('1970-01-01','0.0.0.0'):NULL} diff --git a/tests/queries/0_stateless/03222_json_empty_as_default.sql b/tests/queries/0_stateless/03222_json_empty_as_default.sql new file mode 100644 index 00000000000..1243d450c2e --- /dev/null +++ b/tests/queries/0_stateless/03222_json_empty_as_default.sql @@ -0,0 +1,60 @@ +SET input_format_json_empty_as_default = 1, allow_experimental_variant_type = 1; + +-- Simple types +-- { echoOn } +SELECT x FROM format(JSONEachRow, 'x Date', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x Date32', '{"x":""}'); +SELECT toTimeZone(x, 'UTC') FROM format(JSONEachRow, 'x DateTime', '{"x":""}'); +SELECT toTimeZone(x, 'UTC') FROM format(JSONEachRow, 'x DateTime64', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x IPv4', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x IPv6', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x UUID', '{"x":""}'); +-- { echoOff } + +-- Simple type AggregateFunction +DROP TABLE IF EXISTS table1; +CREATE TABLE table1(col AggregateFunction(uniq, UInt64)) ENGINE=Memory(); +DROP TABLE IF EXISTS table2; +CREATE TABLE table2(UserID UInt64) ENGINE=Memory(); + +INSERT INTO table1 SELECT uniqState(UserID) FROM table2; +INSERT INTO table1 SELECT x FROM format(JSONEachRow, 'x AggregateFunction(uniq, UInt64)' AS T, '{"x":""}'); + +-- { echoOn } +SELECT COUNT(DISTINCT col) FROM table1; +-- { echoOff } + +DROP TABLE table1; +DROP TABLE table2; + +-- The setting input_format_defaults_for_omitted_fields determines the default value if enabled. +CREATE TABLE table1(address IPv6 DEFAULT toIPv6('2001:db8:3333:4444:5555:6666:7777:8888')) ENGINE=Memory(); + +SET input_format_defaults_for_omitted_fields = 0; +INSERT INTO table1 FORMAT JSONEachRow {"address":""}; + +SET input_format_defaults_for_omitted_fields = 1; +INSERT INTO table1 FORMAT JSONEachRow {"address":""}; + +-- { echoOn } +SELECT * FROM table1 ORDER BY address ASC; +-- { echoOff } + +DROP TABLE table1; + +-- Nullable +-- { echoOn } +SELECT x FROM format(JSONEachRow, 'x Nullable(IPv6)', '{"x":""}'); + +-- Compound types +SELECT x FROM format(JSONEachRow, 'x Array(UUID)', '{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e",""]}'); +SELECT x FROM format(JSONEachRow, 'x Array(Nullable(IPv6))', '{"x":["",""]}'); +SELECT x FROM format(JSONEachRow, 'x Tuple(Date, IPv4, String)', '{"x":["", "", "abc"]}'); +SELECT x FROM format(JSONEachRow, 'x Map(String, IPv6)', '{"x":{"abc": ""}}'); +SELECT x FROM format(JSONEachRow, 'x Variant(Date, UUID)', '{"x":""}'); + +-- Deep composition +SELECT x FROM format(JSONEachRow, 'x Array(Array(IPv6))', '{"x":[["2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF", ""], ["", "2001:db8:3333:4444:5555:6666:7777:8888"]]}'); +SELECT x FROM format(JSONEachRow, 'x Variant(Date, Array(UUID))', '{"x":["", "b15f852c-c41a-4fd6-9247-1929c841715e"]}'); +SELECT x FROM format(JSONEachRow, 'x Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))', '{"x":[[""], ["",{"abc":""}]]}'); +SELECT x FROM format(JSONEachRow, 'x Map(Tuple(Date,IPv4), Variant(UUID,IPv6))', '{"x":{["",""]:""}}'); diff --git a/tests/queries/0_stateless/03222_json_empty_as_default_small_read_buffer.reference b/tests/queries/0_stateless/03222_json_empty_as_default_small_read_buffer.reference new file mode 100644 index 00000000000..8176d7895d8 --- /dev/null +++ b/tests/queries/0_stateless/03222_json_empty_as_default_small_read_buffer.reference @@ -0,0 +1,8 @@ +Array(UUID) +{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e","00000000-0000-0000-0000-000000000000"]} +{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e","00000000-0000-0000-0000-000000000000"]} +{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e","00000000-0000-0000-0000-000000000000"]} +Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6))) +{"x":[["00000000-0000-0000-0000-000000000000"],["00000000-0000-0000-0000-000000000000",{"abc":"::"}]]} +{"x":[["00000000-0000-0000-0000-000000000000"],["00000000-0000-0000-0000-000000000000",{"abc":"::"}]]} +{"x":[["00000000-0000-0000-0000-000000000000"],["00000000-0000-0000-0000-000000000000",{"abc":"::"}]]} diff --git a/tests/queries/0_stateless/03222_json_empty_as_default_small_read_buffer.sh b/tests/queries/0_stateless/03222_json_empty_as_default_small_read_buffer.sh new file mode 100755 index 00000000000..6b69fb2e9dc --- /dev/null +++ b/tests/queries/0_stateless/03222_json_empty_as_default_small_read_buffer.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DATA_FILE=$CLICKHOUSE_TEST_UNIQUE_NAME.json + +# Wrapper for clickhouse-client to always output in JSONEachRow format, that +# way format settings will not affect output. +function clickhouse_local() +{ + $CLICKHOUSE_LOCAL --output-format JSONEachRow "$@" +} + +echo 'Array(UUID)' +echo '{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e",""]}' > $DATA_FILE +# Use increasingly smaller read buffers. +clickhouse_local -q "SELECT x FROM file('$DATA_FILE', 'JSONEachRow', 'x Array(UUID)') SETTINGS input_format_json_empty_as_default=1, input_format_parallel_parsing=0, storage_file_read_method='read', max_read_buffer_size=4" +clickhouse_local -q "SELECT x FROM file('$DATA_FILE', 'JSONEachRow', 'x Array(UUID)') SETTINGS input_format_json_empty_as_default=1, input_format_parallel_parsing=0, storage_file_read_method='read', max_read_buffer_size=2" +clickhouse_local -q "SELECT x FROM file('$DATA_FILE', 'JSONEachRow', 'x Array(UUID)') SETTINGS input_format_json_empty_as_default=1, input_format_parallel_parsing=0, storage_file_read_method='read', max_read_buffer_size=1" + +echo 'Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))' +echo '{"x":[[""], ["",{"abc":""}]]}' > $DATA_FILE +# Use increasingly smaller read buffers. +clickhouse_local -q "SELECT x FROM file('$DATA_FILE', 'JSONEachRow', 'x Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))') SETTINGS input_format_json_empty_as_default=1, input_format_parallel_parsing=0, storage_file_read_method='read', max_read_buffer_size=16" +clickhouse_local -q "SELECT x FROM file('$DATA_FILE', 'JSONEachRow', 'x Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))') SETTINGS input_format_json_empty_as_default=1, input_format_parallel_parsing=0, storage_file_read_method='read', max_read_buffer_size=8" +clickhouse_local -q "SELECT x FROM file('$DATA_FILE', 'JSONEachRow', 'x Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))') SETTINGS input_format_json_empty_as_default=1, input_format_parallel_parsing=0, storage_file_read_method='read', max_read_buffer_size=1" + +rm $DATA_FILE diff --git a/tests/queries/0_stateless/03230_array_zip_unaligned.reference b/tests/queries/0_stateless/03230_array_zip_unaligned.reference index 7067f8788e5..d373cf47a9c 100644 --- a/tests/queries/0_stateless/03230_array_zip_unaligned.reference +++ b/tests/queries/0_stateless/03230_array_zip_unaligned.reference @@ -1,5 +1,6 @@ [('a','d'),('b','e'),('c','f')] Array(Tuple(Nullable(String), Nullable(String))) [('a','d','g'),('b','e','h'),('c','f','i')] +[] [('a','d'),('b','e'),('c','f'),(NULL,'g')] [('a',1),(NULL,2),(NULL,3)] [('a',1,1.1),('b',2,2.2),('c',NULL,3.3),(NULL,NULL,4.4)] diff --git a/tests/queries/0_stateless/03230_array_zip_unaligned.sql b/tests/queries/0_stateless/03230_array_zip_unaligned.sql index 90b7aa47bfd..08d77737e54 100644 --- a/tests/queries/0_stateless/03230_array_zip_unaligned.sql +++ b/tests/queries/0_stateless/03230_array_zip_unaligned.sql @@ -2,7 +2,7 @@ SELECT arrayZipUnaligned(['a', 'b', 'c'], ['d', 'e', 'f']) as x, toTypeName(x); SELECT arrayZipUnaligned(['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']); -SELECT arrayZipUnaligned(); -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION } +SELECT arrayZipUnaligned(); SELECT arrayZipUnaligned('a', 'b', 'c'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/03231_prewhere_conditions_order.reference b/tests/queries/0_stateless/03231_prewhere_conditions_order.reference new file mode 100644 index 00000000000..bb14c5f88f2 --- /dev/null +++ b/tests/queries/0_stateless/03231_prewhere_conditions_order.reference @@ -0,0 +1 @@ +1 [0,1] [0,1] diff --git a/tests/queries/0_stateless/03231_prewhere_conditions_order.sql b/tests/queries/0_stateless/03231_prewhere_conditions_order.sql new file mode 100644 index 00000000000..acaba12684c --- /dev/null +++ b/tests/queries/0_stateless/03231_prewhere_conditions_order.sql @@ -0,0 +1,6 @@ +drop table if exists test; +create table test (x UInt32, arr1 Array(UInt32), arr2 Array(UInt32)) engine=MergeTree order by x; +insert into test values (1, [0, 1], [0, 1]), (2, [0], [0, 1]); +select * from test where x == 1 and arrayExists((x1, x2) -> (x1 == x2), arr1, arr2); +drop table test; + diff --git a/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference b/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference new file mode 100644 index 00000000000..cad1bf13574 --- /dev/null +++ b/tests/queries/0_stateless/03231_restore_user_with_existing_role.reference @@ -0,0 +1,6 @@ +Everything dropped +User dropped +Nothing dropped +Nothing dropped, mode=replace +Nothing dropped, mode=create +ACCESS_ENTITY_ALREADY_EXISTS diff --git a/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh b/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh new file mode 100755 index 00000000000..04f907b719d --- /dev/null +++ b/tests/queries/0_stateless/03231_restore_user_with_existing_role.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +# Disabled parallel since RESTORE can only restore either all users or no users +# (it can't restore only users added by the current test run), +# so a RESTORE from a parallel test run could recreate our users before we expect that. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user_a="user_a_${CLICKHOUSE_TEST_UNIQUE_NAME}" +role_b="role_b_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +${CLICKHOUSE_CLIENT} -m --query " +CREATE ROLE ${role_b} SETTINGS custom_x=1; +CREATE USER ${user_a} DEFAULT ROLE ${role_b} SETTINGS custom_x=2; +" + +backup_name="Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}')" + +${CLICKHOUSE_CLIENT} --query "BACKUP TABLE system.users, TABLE system.roles TO ${backup_name} FORMAT Null" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" + +do_check() +{ + local replacements + replacements="s/${user_a}/user_a/g; s/${role_b}/role_b/g" + local check_info + check_info=$(${CLICKHOUSE_CLIENT} -mq " + SHOW CREATE USER ${user_a}; + SHOW GRANTS FOR ${user_a}; + SHOW CREATE ROLE ${role_b}; + SHOW GRANTS FOR ${role_b}; + " | sed "${replacements}") + local expected + expected=$'CREATE USER user_a IDENTIFIED WITH no_password DEFAULT ROLE role_b SETTINGS custom_x = 2\nGRANT role_b TO user_a\nCREATE ROLE role_b SETTINGS custom_x = 1' + if [[ "${check_info}" != "${expected}" ]]; then + echo "Assertion failed:" + echo "\"${check_info}\"" + echo "!=" + echo "\"${expected}\"" + echo "Test database: ${CLICKHOUSE_DATABASE}" >&2 + fi +} + +echo "Everything dropped" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}" +${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" +do_check + +echo "User dropped" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user_a}" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" +do_check + +# TODO: Cannot restore a dropped role granted to an existing user. The result after RESTORE ALL below is the following: +# CREATE USER user_a DEFAULT ROLE NONE SETTINGS custom_x = 2; GRANT NONE TO user_a; CREATE ROLE role_b SETTINGS custom_x = 1 +# because `role_b` is restored but not granted to existing user `user_a`. +# +# echo "Role dropped" +# ${CLICKHOUSE_CLIENT} --query "DROP ROLE ${role_b}" +# ${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" +# do_check + +echo "Nothing dropped" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} FORMAT Null" +do_check + +echo "Nothing dropped, mode=replace" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} SETTINGS create_access='replace' FORMAT Null" +do_check + +echo "Nothing dropped, mode=create" +${CLICKHOUSE_CLIENT} --query "RESTORE ALL FROM ${backup_name} SETTINGS create_access='create' FORMAT Null" 2>&1 | grep -om1 "ACCESS_ENTITY_ALREADY_EXISTS" +do_check diff --git a/tests/queries/1_stateful/00091_prewhere_two_conditions.sql b/tests/queries/1_stateful/00091_prewhere_two_conditions.sql index cd88743160c..649c63f2ec9 100644 --- a/tests/queries/1_stateful/00091_prewhere_two_conditions.sql +++ b/tests/queries/1_stateful/00091_prewhere_two_conditions.sql @@ -7,7 +7,7 @@ SET optimize_move_to_prewhere = 1; SET enable_multiple_prewhere_read_steps = 1; SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00'; -SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00'; +SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND URL != ''; SELECT uniq(*) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND EventDate = '2014-03-21'; WITH toTimeZone(EventTime, 'Asia/Dubai') AS xyz SELECT uniq(*) FROM test.hits WHERE xyz >= '2014-03-20 00:00:00' AND xyz < '2014-03-21 00:00:00' AND EventDate = '2014-03-21';