diff --git a/base/common/ReadlineLineReader.cpp b/base/common/ReadlineLineReader.cpp index d52ac0e9769..095df319acc 100644 --- a/base/common/ReadlineLineReader.cpp +++ b/base/common/ReadlineLineReader.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace { @@ -107,6 +109,8 @@ ReadlineLineReader::ReadlineLineReader( throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + strerror(errno)); rl_variable_bind("completion-ignore-case", "on"); + // TODO: it doesn't work + // history_write_timestamps = 1; } ReadlineLineReader::~ReadlineLineReader() @@ -129,6 +133,11 @@ LineReader::InputStatus ReadlineLineReader::readOneLine(const String & prompt) void ReadlineLineReader::addToHistory(const String & line) { add_history(line.c_str()); + + // Flush changes to the disk + // NOTE readline builds a buffer of all the lines to write, and write them in one syscall. + // Thus there is no need to lock the history file here. + write_history(history_file_path.c_str()); } #if RL_VERSION_MAJOR >= 7 diff --git a/benchmark/monetdb/usability.md b/benchmark/monetdb/usability.md index 20eaf37f9e8..741079b5663 100644 --- a/benchmark/monetdb/usability.md +++ b/benchmark/monetdb/usability.md @@ -2,7 +2,7 @@ Go to https://www.monetdb.org/ The graphical design of the website is a bit old-fashioned but I do not afraid. -Dowload now. +Download now. Latest binary releases. Ubuntu & Debian. @@ -1103,7 +1103,7 @@ Ok, it's doing something at least for twenty minues... clk: 28:02 min ``` -Finally it has loaded data successfuly in 28 minutes. It's not fast - just below 60 000 rows per second. +Finally it has loaded data successfully in 28 minutes. It's not fast - just below 60 000 rows per second. But the second query from the test does not work: diff --git a/cmake/find/cassandra.cmake b/cmake/find/cassandra.cmake index f41e0f645f4..6ff7697ea57 100644 --- a/cmake/find/cassandra.cmake +++ b/cmake/find/cassandra.cmake @@ -1,6 +1,10 @@ option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES}) if (ENABLE_CASSANDRA) + if (APPLE) + SET(CMAKE_MACOSX_RPATH ON) + endif() + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") message (ERROR "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 7504b01d695..3c5b6edc05d 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -122,11 +122,11 @@ initdb() CLICKHOUSE_DATADIR_FROM_CONFIG=$CLICKHOUSE_DATADIR fi - if ! getent group ${CLICKHOUSE_USER} >/dev/null; then + if ! getent passwd ${CLICKHOUSE_USER} >/dev/null; then echo "Can't chown to non-existing user ${CLICKHOUSE_USER}" return fi - if ! getent passwd ${CLICKHOUSE_GROUP} >/dev/null; then + if ! getent group ${CLICKHOUSE_GROUP} >/dev/null; then echo "Can't chown to non-existing group ${CLICKHOUSE_GROUP}" return fi diff --git a/docker/images.json b/docker/images.json index 3eeff6b7c00..0ab1688efb0 100644 --- a/docker/images.json +++ b/docker/images.json @@ -2,7 +2,6 @@ "docker/packager/deb": { "name": "yandex/clickhouse-deb-builder", "dependent": [ - "docker/packager/unbundled", "docker/test/stateless", "docker/test/stateless_with_coverage", "docker/test/stateless_pytest", @@ -16,10 +15,6 @@ "docker/test/pvs" ] }, - "docker/packager/unbundled": { - "name": "yandex/clickhouse-unbundled-builder", - "dependent": [] - }, "docker/test/coverage": { "name": "yandex/clickhouse-coverage", "dependent": [] @@ -97,6 +92,10 @@ "name": "yandex/clickhouse-fasttest", "dependent": [] }, + "docker/test/style": { + "name": "yandex/clickhouse-style-test", + "dependent": [] + }, "docker/test/integration/s3_proxy": { "name": "yandex/clickhouse-s3-proxy", "dependent": [] diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index b1f711509fb..c1260b5c7ff 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,9 +1,9 @@ # docker build -t yandex/clickhouse-deb-builder . -FROM ubuntu:20.04 +FROM ubuntu:19.10 RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main" >> /etc/apt/sources.list +RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list # initial packages RUN apt-get --allow-unauthenticated update -y \ @@ -25,17 +25,13 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin + +# Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ - gcc-10 \ - g++-10 \ gcc-9 \ g++-9 \ - llvm-11 \ - clang-11 \ - lld-11 \ - clang-tidy-11 \ llvm-10 \ clang-10 \ lld-10 \ @@ -43,19 +39,54 @@ RUN apt-get --allow-unauthenticated update -y \ clang-9 \ lld-9 \ clang-tidy-9 \ + libicu-dev \ + libreadline-dev \ + gperf \ ninja-build \ perl \ pkg-config \ devscripts \ debhelper \ git \ + libc++-dev \ + libc++abi-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-thread-dev \ + libboost-iostreams-dev \ + libboost-regex-dev \ + zlib1g-dev \ + liblz4-dev \ + libdouble-conversion-dev \ + librdkafka-dev \ + libpoconetssl62 \ + libpoco-dev \ + libgoogle-perftools-dev \ + libzstd-dev \ + libltdl-dev \ + libre2-dev \ + libjemalloc-dev \ + libmsgpack-dev \ + libcurl4-openssl-dev \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ + unixodbc-dev \ + odbcinst \ tzdata \ gperf \ alien \ + libcapnp-dev \ cmake \ gdb \ + pigz \ moreutils \ - pigz + libcctz-dev \ + libldap2-dev \ + libsasl2-dev \ + heimdal-multidev \ + libhyperscan-dev # This symlink required by gcc to find lld compiler diff --git a/docker/packager/packager b/docker/packager/packager index 63652da2439..bc97429336c 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -11,7 +11,6 @@ SCRIPT_PATH = os.path.realpath(__file__) IMAGE_MAP = { "deb": "yandex/clickhouse-deb-builder", "binary": "yandex/clickhouse-binary-builder", - "unbundled": "yandex/clickhouse-unbundled-builder" } def check_image_exists_locally(image_name): @@ -177,9 +176,7 @@ if __name__ == "__main__": parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-10", "clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", - "clang-11", "clang-11-darwin", "clang-11-aarch64", "clang-11-freebsd", - "gcc-9", "gcc-10"), default="gcc-9") + parser.add_argument("--compiler", choices=("clang-10-darwin", "clang-10-aarch64", "clang-10-freebsd", "gcc-9", "clang-10"), default="gcc-9") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true") @@ -200,7 +197,7 @@ if __name__ == "__main__": if not os.path.isabs(args.output_dir): args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir)) - image_type = 'binary' if args.package_type == 'performance' else 'unbundled' if args.unbundled else args.package_type + image_type = 'binary' if args.package_type == 'performance' else args.package_type image_name = IMAGE_MAP[image_type] if not os.path.isabs(args.clickhouse_repo_path): diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile deleted file mode 100644 index d099be6cb90..00000000000 --- a/docker/packager/unbundled/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# docker build -t yandex/clickhouse-unbundled-builder . -FROM yandex/clickhouse-deb-builder - -# Libraries from OS are only needed to test the "unbundled" build (that is not used in production). -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - libicu-dev \ - libreadline-dev \ - gperf \ - perl \ - pkg-config \ - devscripts \ - libc++-dev \ - libc++abi-dev \ - libboost-program-options-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-thread-dev \ - libboost-iostreams-dev \ - libboost-regex-dev \ - zlib1g-dev \ - liblz4-dev \ - libdouble-conversion-dev \ - librdkafka-dev \ - libpoconetssl62 \ - libpoco-dev \ - libgoogle-perftools-dev \ - libzstd-dev \ - libltdl-dev \ - libre2-dev \ - libjemalloc-dev \ - libmsgpack-dev \ - libcurl4-openssl-dev \ - opencl-headers \ - ocl-icd-libopencl1 \ - intel-opencl-icd \ - unixodbc-dev \ - odbcinst \ - tzdata \ - gperf \ - alien \ - libcapnp-dev \ - cmake \ - gdb \ - pigz \ - moreutils \ - libcctz-dev \ - libldap2-dev \ - libsasl2-dev \ - heimdal-multidev \ - libhyperscan-dev - -COPY build.sh / - -CMD ["/bin/bash", "/build.sh"] diff --git a/docker/packager/unbundled/build.sh b/docker/packager/unbundled/build.sh deleted file mode 100755 index 9fd246bcc33..00000000000 --- a/docker/packager/unbundled/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -x -e - -# Update tzdata to the latest version. It is embedded into clickhouse binary. -sudo apt-get update && sudo apt-get install tzdata - -ccache --show-stats ||: -ccache --zero-stats ||: -build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S' -mv /*.deb /output -mv *.changes /output -mv *.buildinfo /output -mv /*.rpm /output ||: # if exists -mv /*.tgz /output ||: # if exists - -ccache --show-stats ||: -ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index fd8a7099b60..faadfca1210 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -158,6 +158,8 @@ TESTS_TO_SKIP=( 01280_ssd_complex_key_dictionary 00652_replicated_mutations_zookeeper 01411_bayesian_ab_testing + 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently + 01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently ) clickhouse-test -j 4 --no-long --testname --shard --zookeeper --skip ${TESTS_TO_SKIP[*]} 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt diff --git a/docker/test/stress/stress b/docker/test/stress/stress index b107dc59829..173cb8442d6 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -24,20 +24,26 @@ def run_perf_test(cmd, xmls_path, output_folder): return p +def get_options(i): + options = "" + if 0 < i: + options += " --order=random" + if i == 1: + options += " --atomic-db-engine" + return options + + def run_func_test(cmd, output_prefix, num_processes, skip_tests_option): skip_list_opt = get_skip_list_cmd(cmd) output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)] - f = open(output_paths[0], 'w') - main_command = "{} {} {}".format(cmd, skip_list_opt, skip_tests_option) - logging.info("Run func tests main cmd '%s'", main_command) - pipes = [Popen(main_command, shell=True, stdout=f, stderr=f)] - for output_path in output_paths[1:]: - time.sleep(0.5) - f = open(output_path, 'w') - full_command = "{} {} --order=random {}".format(cmd, skip_list_opt, skip_tests_option) + pipes = [] + for i in range(0, len(output_paths)): + f = open(output_paths[i], 'w') + full_command = "{} {} {} {}".format(cmd, skip_list_opt, get_options(i), skip_tests_option) logging.info("Run func tests '%s'", full_command) p = Popen(full_command, shell=True, stdout=f, stderr=f) pipes.append(p) + time.sleep(0.5) return pipes diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile new file mode 100644 index 00000000000..1a4356caced --- /dev/null +++ b/docker/test/style/Dockerfile @@ -0,0 +1,8 @@ +# docker build -t yandex/clickhouse-style-test . +FROM ubuntu:20.04 + +RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes shellcheck libxml2-utils git python3-pip && pip3 install codespell + + +CMD cd /ClickHouse/utils/check-style && ./check-style -n | tee /test_output/style_output.txt && \ + ./check-duplicate-includes.sh | tee /test_output/duplicate_output.txt diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md index 684e7e28112..109ae6c4601 100644 --- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md @@ -31,7 +31,7 @@ For a description of request parameters, see [statement description](../../../sq **ReplacingMergeTree Parameters** -- `ver` — column with version. Type `UInt*`, `Date` or `DateTime`. Optional parameter. +- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter. When merging, `ReplacingMergeTree` from all the rows with the same sorting key leaves only one: diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md index deac786bc50..c637ef03f71 100644 --- a/docs/en/operations/quotas.md +++ b/docs/en/operations/quotas.md @@ -8,7 +8,7 @@ toc_title: Quotas Quotas allow you to limit resource usage over a period of time or track the use of resources. Quotas are set up in the user config, which is usually ‘users.xml’. -The system also has a feature for limiting the complexity of a single query. See the section “Restrictions on query complexity”). +The system also has a feature for limiting the complexity of a single query. See the section [Restrictions on query complexity](../operations/settings/query-complexity.md). In contrast to query complexity restrictions, quotas: diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index c49ead23c09..80d4d659bd3 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -399,7 +399,7 @@ The cache is shared for the server and memory is allocated as needed. The cache ``` ## max\_server\_memory\_usage {#max_server_memory_usage} -Limits total RAM usage by the ClickHouse server. You can specify it only for the default profile. +Limits total RAM usage by the ClickHouse server. Possible values: diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md b/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md new file mode 100644 index 00000000000..f0a969a8530 --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md @@ -0,0 +1,85 @@ +--- +toc_priority: 114 +--- + +# groupArraySample {#grouparraysample} + +Creates an array of sample argument values. The size of the resulting array is limited to `max_size` elements. Argument values are selected and added to the array randomly. + +**Syntax** + +``` sql +groupArraySample(max_size)(x) +``` +or +``` sql +groupArraySample(max_size, seed)(x) +``` + +**Parameters** + +- `max_size` — Maximum size of the resulting array. Positive [UInt64](../../data-types/int-uint.md). +- `seed` — Seed for the random number generator. Optional, can be omitted. Positive [UInt64](../../data-types/int-uint.md). Default value: `123456`. +- `x` — Argument name. [String](../../data-types/string.md). + +**Returned values** + +- Array of randomly selected `x` arguments. + +Type: [Array](../../data-types/array.md). + +**Examples** + +Consider table `colors`: + +``` text +┌─id─┬─color──┐ +│ 1 │ red │ +│ 2 │ blue │ +│ 3 │ green │ +│ 4 │ white │ +│ 5 │ orange │ +└────┴────────┘ +``` + +Select `id`-s query: + +``` sql +SELECT groupArraySample(3)(id) FROM colors; +``` + +Result: + +``` text +┌─groupArraySample(3)(id)─┐ +│ [1,2,4] │ +└─────────────────────────┘ +``` + +Select `color`-s query: + +``` sql +SELECT groupArraySample(3)(color) FROM colors; +``` + +Result: + +```text +┌─groupArraySample(3)(color)─┐ +│ ['white','blue','green'] │ +└────────────────────────────┘ +``` + +Select `color`-s query with different seed: + +``` sql +SELECT groupArraySample(3, 987654321)(color) FROM colors; +``` + +Result: + +```text +┌─groupArraySample(3, 987654321)(color)─┐ +│ ['red','orange','green'] │ +└───────────────────────────────────────┘ +``` diff --git a/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md index a1e95c5b5f4..cb3c6aea34b 100644 --- a/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/es/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,7 +33,7 @@ Para obtener una descripción de los parámetros de solicitud, consulte [descrip **ReplacingMergeTree Parámetros** -- `ver` — column with version. Type `UInt*`, `Date` o `DateTime`. Parámetro opcional. +- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` o `DateTime64`. Parámetro opcional. Al fusionar, `ReplacingMergeTree` de todas las filas con la misma clave primaria deja solo una: diff --git a/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md index 0ace0e05afc..4ece20461cb 100644 --- a/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/fa/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **پارامترهای جایگزین** -- `ver` — column with version. Type `UInt*`, `Date` یا `DateTime`. پارامتر اختیاری. +- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` یا `DateTime64`. پارامتر اختیاری. هنگام ادغام, `ReplacingMergeTree` از تمام ردیف ها با همان کلید اصلی تنها یک برگ دارد: diff --git a/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md index ac3c0f3b021..755249c1a38 100644 --- a/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/fr/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,7 +33,7 @@ Pour une description des paramètres de requête, voir [demande de description]( **ReplacingMergeTree Paramètres** -- `ver` — column with version. Type `UInt*`, `Date` ou `DateTime`. Paramètre facultatif. +- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` ou `DateTime64`. Paramètre facultatif. Lors de la fusion, `ReplacingMergeTree` de toutes les lignes avec la même clé primaire ne laisse qu'un: diff --git a/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md index c3df9559415..e2cce893e3a 100644 --- a/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ja/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **ReplacingMergeTreeパラメータ** -- `ver` — column with version. Type `UInt*`, `Date` または `DateTime`. 任意パラメータ。 +- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` または `DateTime64`. 任意パラメータ。 マージ時, `ReplacingMergeTree` 同じ主キーを持つすべての行から、一つだけを残します: diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 025772ea3ca..6775b5378cb 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -1,3 +1,5 @@ +# Инструкция для разработчиков + Сборка ClickHouse поддерживается на Linux, FreeBSD, Mac OS X. # Если вы используете Windows {#esli-vy-ispolzuete-windows} diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md index 4aa1eb556f3..fefc3c65b38 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md @@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **Параметры ReplacingMergeTree** -- `ver` — столбец с версией, тип `UInt*`, `Date` или `DateTime`. Необязательный параметр. +- `ver` — столбец с версией, тип `UInt*`, `Date`, `DateTime` или `DateTime64`. Необязательный параметр. При слиянии, из всех строк с одинаковым значением ключа сортировки `ReplacingMergeTree` оставляет только одну: diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 0c4389fe0c1..63319ed2c01 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -374,7 +374,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ## max_server_memory_usage {#max_server_memory_usage} -Ограничивает объём оперативной памяти, используемой сервером ClickHouse. Настройка может быть задана только для профиля `default`. +Ограничивает объём оперативной памяти, используемой сервером ClickHouse. Возможные значения: diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index d222fd4e284..fa371df2d4a 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -637,7 +637,7 @@ Upd. Готово (все директории кроме contrib). Требует 7.26. Коллеги начали делать, есть результат. Upd. В Аркадии частично работает небольшая часть тестов. И этого достаточно. -### 7.29. Опции clickhouse install, stop, start вместо postinst, init.d, systemd скриптов {#optsii-clickhouse-install-stop-start-vmesto-postinst-init-d-systemd-skriptov} +### 7.29. + Опции clickhouse install, stop, start вместо postinst, init.d, systemd скриптов {#optsii-clickhouse-install-stop-start-vmesto-postinst-init-d-systemd-skriptov} Низкий приоритет. @@ -786,7 +786,7 @@ Upd. Готово. Павел Круглов, ВШЭ и Яндекс. Есть pull request. Готово. -### 8.17. ClickHouse как MySQL реплика {#clickhouse-kak-mysql-replika} +### 8.17. + ClickHouse как MySQL реплика {#clickhouse-kak-mysql-replika} Задачу делает BohuTANG. @@ -1447,11 +1447,11 @@ Upd. Возможно будет отложено на следующий год Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ. В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса. -+ 1. Оптимизация top sort. +\+ 1. Оптимизация top sort. В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top. -+ 2. Рекурсивный вариант сортировки по кортежам. +\+ 2. Рекурсивный вариант сортировки по кортежам. Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения. @@ -1583,8 +1583,8 @@ Upd. Готово. После 10.14. -[\#7237](https://github.com/ClickHouse/ClickHouse/issues/7237) -[\#2655](https://github.com/ClickHouse/ClickHouse/issues/2655) +[#7237](https://github.com/ClickHouse/ClickHouse/issues/7237) +[#2655](https://github.com/ClickHouse/ClickHouse/issues/2655) ### 22.23. Правильная обработка Nullable в функциях, которые кидают исключение на default значении: modulo, intDiv {#pravilnaia-obrabotka-nullable-v-funktsiiakh-kotorye-kidaiut-iskliuchenie-na-default-znachenii-modulo-intdiv} @@ -1598,7 +1598,7 @@ Upd. Готово. ### 22.26. Плохая производительность quantileTDigest {#plokhaia-proizvoditelnost-quantiletdigest} -[\#2668](https://github.com/ClickHouse/ClickHouse/issues/2668) +[#2668](https://github.com/ClickHouse/ClickHouse/issues/2668) Алексей Миловидов или будет переназначено. diff --git a/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md index a24c84e9a16..f586b97cb2f 100644 --- a/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/tr/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,7 +33,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **ReplacingMergeTree Parametreleri** -- `ver` — column with version. Type `UInt*`, `Date` veya `DateTime`. İsteğe bağlı parametre. +- `ver` — column with version. Type `UInt*`, `Date`, `DateTime` veya `DateTime64`. İsteğe bağlı parametre. Birleş whenirken, `ReplacingMergeTree` aynı birincil anahtara sahip tüm satırlardan sadece bir tane bırakır: diff --git a/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md index 626597eeaf0..03b47172400 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md @@ -25,7 +25,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] **参数** -- `ver` — 版本列。类型为 `UInt*`, `Date` 或 `DateTime`。可选参数。 +- `ver` — 版本列。类型为 `UInt*`, `Date`, `DateTime` 或 `DateTime64`。可选参数。 合并的时候，`ReplacingMergeTree` 从所有具有相同主键的行中选择一行留下： - 如果 `ver` 列未指定，选择最后一条。 diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index ca348382281..5cdf5766a44 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -247,12 +247,15 @@ try context->setCurrentDatabase(default_database); applyCmdOptions(); - if (!context->getPath().empty()) + String path = context->getPath(); + if (!path.empty()) { /// Lock path directory before read status.emplace(context->getPath() + "status", StatusFile::write_full_info); - LOG_DEBUG(log, "Loading metadata from {}", context->getPath()); + LOG_DEBUG(log, "Loading metadata from {}", path); + Poco::File(path + "data/").createDirectories(); + Poco::File(path + "metadata/").createDirectories(); loadMetadataSystem(*context); attachSystemTables(*context); loadMetadata(*context); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ddc5ec080fb..0298528424c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -356,7 +356,7 @@ int Server::main(const std::vector & /*args*/) std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); std::string default_database = config().getString("default_database", "default"); - /// Check that the process' user id matches the owner of the data. + /// Check that the process user id matches the owner of the data. const auto effective_user_id = geteuid(); struct stat statbuf; if (stat(path.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) @@ -468,6 +468,9 @@ int Server::main(const std::vector & /*args*/) } { + Poco::File(path + "data/").createDirectories(); + Poco::File(path + "metadata/").createDirectories(); + /// Directory with metadata of tables, which was marked as dropped by Atomic database Poco::File(path + "metadata_dropped/").createDirectories(); } diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index cb5f6604a93..d9c619809fc 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -388,13 +388,14 @@ public: { for (size_t j = 0; j < UNROLL_COUNT; ++j) { - if (has_data[j * 256 + k]) + size_t idx = j * 256 + k; + if (has_data[idx]) { AggregateDataPtr & place = map[k]; if (unlikely(!place)) init(place); - func.merge(place + place_offset, reinterpret_cast(&places[256 * j + k]), arena); + func.merge(place + place_offset, reinterpret_cast(&places[idx]), nullptr); } } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f8359c4c93..52d7211f20e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -140,6 +140,7 @@ endmacro() add_object_library(clickhouse_access Access) add_object_library(clickhouse_core Core) +add_object_library(clickhouse_core_mysql Core/MySQL) add_object_library(clickhouse_compression Compression) add_object_library(clickhouse_datastreams DataStreams) add_object_library(clickhouse_datatypes DataTypes) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 7e4e6233f73..58543d6a4dd 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -31,8 +31,17 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; extern const int LOGICAL_ERROR; + extern const int TOO_LARGE_ARRAY_SIZE; } +/** Obtaining array as Field can be slow for large arrays and consume vast amount of memory. + * Just don't allow to do it. + * You can increase the limit if the following query: + * SELECT range(10000000) + * will take less than 500ms on your machine. + */ +static constexpr size_t max_array_size_as_field = 1000000; + ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) : data(std::move(nested_column)), offsets(std::move(offsets_column)) @@ -117,6 +126,11 @@ Field ColumnArray::operator[](size_t n) const { size_t offset = offsetAt(n); size_t size = sizeAt(n); + + if (size > max_array_size_as_field) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array of size {} is too large to be manipulated as single field, maximum size {}", + size, max_array_size_as_field); + Array res(size); for (size_t i = 0; i < size; ++i) @@ -130,6 +144,11 @@ void ColumnArray::get(size_t n, Field & res) const { size_t offset = offsetAt(n); size_t size = sizeAt(n); + + if (size > max_array_size_as_field) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array of size {} is too large to be manipulated as single field, maximum size {}", + size, max_array_size_as_field); + res = Array(size); Array & res_arr = DB::get(res); diff --git a/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp deleted file mode 100644 index de60d382301..00000000000 --- a/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ /dev/null @@ -1,229 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include - - -using namespace DB; - - -template -auto getZooKeeper(Args &&... args) -{ - /// In our CI infrastructure it is typical that ZooKeeper is unavailable for some amount of time. - size_t i; - for (i = 0; i < 100; ++i) - { - try - { - auto zookeeper = std::make_unique("localhost:2181", std::forward(args)...); - zookeeper->exists("/"); - zookeeper->createIfNotExists("/clickhouse_test", "Unit tests of ClickHouse"); - return zookeeper; - } - catch (...) - { - std::cerr << "Zookeeper is unavailable, try " << i << std::endl; - sleep(1); - continue; - } - } - - std::cerr << "No zookeeper after " << i << " tries. skip tests." << std::endl; - exit(0); -} - - -TEST(zkutil, MultiNiceExceptionMsg) -{ - auto zookeeper = getZooKeeper(); - - Coordination::Requests ops; - - ASSERT_NO_THROW( - zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi"); - - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "_", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent)); - zookeeper->multi(ops); - ); - - try - { - ops.clear(); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/c", "_", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeRemoveRequest("/clickhouse_test/zkutil_multi/c", -1)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "BadBoy", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/b", "_", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent)); - - zookeeper->multi(ops); - FAIL(); - } - catch (...) - { - zookeeper->tryRemoveRecursive("/clickhouse_test/zkutil_multi"); - - String msg = getCurrentExceptionMessage(false); - - bool msg_has_reqired_patterns = msg.find("#2") != std::string::npos; - EXPECT_TRUE(msg_has_reqired_patterns) << msg; - } -} - - -TEST(zkutil, MultiAsync) -{ - Coordination::Requests ops; - - getZooKeeper()->tryRemoveRecursive("/clickhouse_test/zkutil_multi"); - - { - ops.clear(); - auto zookeeper = getZooKeeper(); - auto fut = zookeeper->asyncMulti(ops); - } - - { - ops.clear(); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "", zkutil::CreateMode::Persistent)); - - auto zookeeper = getZooKeeper(); - auto fut = zookeeper->tryAsyncMulti(ops); - ops.clear(); - - auto res = fut.get(); - ASSERT_EQ(res.error, Coordination::Error::ZOK); - ASSERT_EQ(res.responses.size(), 2); - } - - EXPECT_ANY_THROW - ( - auto zookeeper = getZooKeeper(); - std::vector> futures; - - for (size_t i = 0; i < 10000; ++i) - { - ops.clear(); - ops.emplace_back(zkutil::makeRemoveRequest("/clickhouse_test/zkutil_multi", -1)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "_", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCheckRequest("/clickhouse_test/zkutil_multi", -1)); - ops.emplace_back(zkutil::makeSetRequest("/clickhouse_test/zkutil_multi", "xxx", 42)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent)); - - futures.emplace_back(zookeeper->asyncMulti(ops)); - } - - futures[0].get(); - ); - - /// Check there are no segfaults for remaining 999 futures - using namespace std::chrono_literals; - std::this_thread::sleep_for(1s); - - try - { - ops.clear(); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi", "_", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest("/clickhouse_test/zkutil_multi/a", "_", zkutil::CreateMode::Persistent)); - - auto zookeeper = getZooKeeper(); - auto fut = zookeeper->tryAsyncMulti(ops); - ops.clear(); - - auto res = fut.get(); - - /// The test is quite heavy. It is normal if session is expired during this test. - /// If we don't check that, the test will be flacky. - if (res.error != Coordination::Error::ZSESSIONEXPIRED && res.error != Coordination::Error::ZCONNECTIONLOSS) - { - ASSERT_EQ(res.error, Coordination::Error::ZNODEEXISTS); - ASSERT_EQ(res.responses.size(), 2); - } - } - catch (const Coordination::Exception & e) - { - if (e.code != Coordination::Error::ZSESSIONEXPIRED && e.code != Coordination::Error::ZCONNECTIONLOSS) - throw; - } -} - -TEST(zkutil, WatchGetChildrenWithChroot) -{ - try - { - const String prefix = "/clickhouse_test/zkutil/watch_get_children_with_chroot"; - - /// Create chroot node firstly - auto zookeeper = getZooKeeper(); - zookeeper->createAncestors(prefix + "/"); - zookeeper = getZooKeeper("", - zkutil::DEFAULT_SESSION_TIMEOUT, - zkutil::DEFAULT_OPERATION_TIMEOUT, - prefix); - - String queue_path = "/queue"; - zookeeper->tryRemoveRecursive(queue_path); - zookeeper->createAncestors(queue_path + "/"); - - zkutil::EventPtr event = std::make_shared(); - zookeeper->getChildren(queue_path, nullptr, event); - { - auto zookeeper2 = getZooKeeper("", - zkutil::DEFAULT_SESSION_TIMEOUT, - zkutil::DEFAULT_OPERATION_TIMEOUT, - prefix); - zookeeper2->create(queue_path + "/children-", "", zkutil::CreateMode::PersistentSequential); - } - event->wait(); - } - catch (...) - { - std::cerr << getCurrentExceptionMessage(true); - throw; - } -} - -TEST(zkutil, MultiCreateSequential) -{ - try - { - const String prefix = "/clickhouse_test/zkutil"; - - /// Create chroot node firstly - auto zookeeper = getZooKeeper(); - zookeeper->createAncestors(prefix + "/"); - zookeeper = getZooKeeper("", - zkutil::DEFAULT_SESSION_TIMEOUT, - zkutil::DEFAULT_OPERATION_TIMEOUT, - "/clickhouse_test"); - - String base_path = "/multi_create_sequential"; - zookeeper->tryRemoveRecursive(base_path); - zookeeper->createAncestors(base_path + "/"); - - Coordination::Requests ops; - String sequential_node_prefix = base_path + "/queue-"; - ops.emplace_back(zkutil::makeCreateRequest(sequential_node_prefix, "", zkutil::CreateMode::EphemeralSequential)); - auto results = zookeeper->multi(ops); - const auto & sequential_node_result_op = dynamic_cast(*results.at(0)); - - EXPECT_FALSE(sequential_node_result_op.path_created.empty()); - EXPECT_GT(sequential_node_result_op.path_created.length(), sequential_node_prefix.length()); - EXPECT_EQ(sequential_node_result_op.path_created.substr(0, sequential_node_prefix.length()), sequential_node_prefix); - } - catch (...) - { - std::cerr << getCurrentExceptionMessage(false); - throw; - } -} - - diff --git a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp index 38f9365ca7f..95b983cc3bb 100644 --- a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp +++ b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp @@ -9,21 +9,34 @@ static bool check() { ThreadPool pool(10); + /// The throwing thread. pool.scheduleOrThrowOnError([] { throw std::runtime_error("Hello, world!"); }); try { - for (size_t i = 0; i < 500; ++i) - pool.scheduleOrThrowOnError([] {}); /// An exception will be rethrown from this method. + while (true) + { + /// An exception from the throwing thread will be rethrown from this method + /// as soon as the throwing thread executed. + + /// This innocent thread may or may not be executed, the following possibilities exist: + /// 1. The throwing thread has already thrown exception and the attempt to schedule the innocent thread will rethrow it. + /// 2. The throwing thread has not executed, the innocent thread will be scheduled and executed. + /// 3. The throwing thread has not executed, the innocent thread will be scheduled but before it will be executed, + /// the throwing thread will be executed and throw exception and it will prevent starting of execution of the innocent thread + /// the method will return and the exception will be rethrown only on call to "wait" or on next call on next loop iteration as (1). + pool.scheduleOrThrowOnError([]{}); + + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } } catch (const std::runtime_error &) { + pool.wait(); return true; } - pool.wait(); - - return false; + __builtin_unreachable(); } diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp new file mode 100644 index 00000000000..b0f5f8ccae2 --- /dev/null +++ b/src/Core/MySQL/Authentication.cpp @@ -0,0 +1,242 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int OPENSSL_ERROR; + extern const int UNKNOWN_EXCEPTION; + extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES; +} + +namespace MySQLProtocol +{ + +using namespace ConnectionPhase; + +namespace Authentication +{ + +static const size_t SCRAMBLE_LENGTH = 20; + +Native41::Native41() +{ + scramble.resize(SCRAMBLE_LENGTH + 1, 0); + Poco::RandomInputStream generator; + + /** Generate a random string using ASCII characters but avoid separator character, + * produce pseudo random numbers between with about 7 bit worth of entropty between 1-127. + * https://github.com/mysql/mysql-server/blob/8.0/mysys/crypt_genhash_impl.cc#L427 + */ + for (size_t i = 0; i < SCRAMBLE_LENGTH; ++i) + { + generator >> scramble[i]; + scramble[i] &= 0x7f; + if (scramble[i] == '\0' || scramble[i] == '$') + scramble[i] = scramble[i] + 1; + } +} + +Native41::Native41(const String & password, const String & auth_plugin_data) +{ + /// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html + /// SHA1( password ) XOR SHA1( "20-bytes random data from server" SHA1( SHA1( password ) ) ) + Poco::SHA1Engine engine1; + engine1.update(password); + const Poco::SHA1Engine::Digest & password_sha1 = engine1.digest(); + + Poco::SHA1Engine engine2; + engine2.update(password_sha1.data(), password_sha1.size()); + const Poco::SHA1Engine::Digest & password_double_sha1 = engine2.digest(); + + Poco::SHA1Engine engine3; + engine3.update(auth_plugin_data.data(), auth_plugin_data.size()); + engine3.update(password_double_sha1.data(), password_double_sha1.size()); + const Poco::SHA1Engine::Digest & digest = engine3.digest(); + + scramble.resize(SCRAMBLE_LENGTH); + for (size_t i = 0; i < SCRAMBLE_LENGTH; i++) + { + scramble[i] = static_cast(password_sha1[i] ^ digest[i]); + } +} + +void Native41::authenticate( + const String & user_name, std::optional auth_response, Context & context, + std::shared_ptr packet_endpoint, bool, const Poco::Net::SocketAddress & address) +{ + if (!auth_response) + { + packet_endpoint->sendPacket(AuthSwitchRequest(getName(), scramble), true); + AuthSwitchResponse response; + packet_endpoint->receivePacket(response); + auth_response = response.value; + } + + if (auth_response->empty()) + { + context.setUser(user_name, "", address); + return; + } + + if (auth_response->size() != Poco::SHA1Engine::DIGEST_SIZE) + throw Exception("Wrong size of auth response. Expected: " + std::to_string(Poco::SHA1Engine::DIGEST_SIZE) + " bytes, received: " + std::to_string(auth_response->size()) + " bytes.", + ErrorCodes::UNKNOWN_EXCEPTION); + + auto user = context.getAccessControlManager().read(user_name); + + Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordDoubleSHA1(); + assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE); + + Poco::SHA1Engine engine; + engine.update(scramble.data(), SCRAMBLE_LENGTH); + engine.update(double_sha1_value.data(), double_sha1_value.size()); + + String password_sha1(Poco::SHA1Engine::DIGEST_SIZE, 0x0); + const Poco::SHA1Engine::Digest & digest = engine.digest(); + for (size_t i = 0; i < password_sha1.size(); i++) + { + password_sha1[i] = digest[i] ^ static_cast((*auth_response)[i]); + } + context.setUser(user_name, password_sha1, address); +} + +#if USE_SSL + +Sha256Password::Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logger * log_) + : public_key(public_key_), private_key(private_key_), log(log_) +{ + /** Native authentication sent 20 bytes + '\0' character = 21 bytes. + * This plugin must do the same to stay consistent with historical behavior if it is set to operate as a default plugin. [1] + * https://github.com/mysql/mysql-server/blob/8.0/sql/auth/sql_authentication.cc#L3994 + */ + scramble.resize(SCRAMBLE_LENGTH + 1, 0); + Poco::RandomInputStream generator; + + for (size_t i = 0; i < SCRAMBLE_LENGTH; ++i) + { + generator >> scramble[i]; + scramble[i] &= 0x7f; + if (scramble[i] == '\0' || scramble[i] == '$') + scramble[i] = scramble[i] + 1; + } +} + +void Sha256Password::authenticate( + const String & user_name, std::optional auth_response, Context & context, + std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) +{ + if (!auth_response) + { + packet_endpoint->sendPacket(AuthSwitchRequest(getName(), scramble), true); + + if (packet_endpoint->in->eof()) + throw Exception("Client doesn't support authentication method " + getName() + " used by ClickHouse. Specifying user password using 'password_double_sha1_hex' may fix the problem.", + ErrorCodes::MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES); + + AuthSwitchResponse response; + packet_endpoint->receivePacket(response); + auth_response.emplace(response.value); + LOG_TRACE(log, "Authentication method mismatch."); + } + else + { + LOG_TRACE(log, "Authentication method match."); + } + + bool sent_public_key = false; + if (auth_response == "\1") + { + LOG_TRACE(log, "Client requests public key."); + BIO * mem = BIO_new(BIO_s_mem()); + SCOPE_EXIT(BIO_free(mem)); + if (PEM_write_bio_RSA_PUBKEY(mem, &public_key) != 1) + { + throw Exception("Failed to write public key to memory. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); + } + char * pem_buf = nullptr; +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wold-style-cast" + int64_t pem_size = BIO_get_mem_data(mem, &pem_buf); +# pragma GCC diagnostic pop + String pem(pem_buf, pem_size); + + LOG_TRACE(log, "Key: {}", pem); + + AuthMoreData data(pem); + packet_endpoint->sendPacket(data, true); + sent_public_key = true; + + AuthSwitchResponse response; + packet_endpoint->receivePacket(response); + auth_response.emplace(response.value); + } + else + { + LOG_TRACE(log, "Client didn't request public key."); + } + + String password; + + /** Decrypt password, if it's not empty. + * The original intention was that the password is a string[NUL] but this never got enforced properly so now we have to accept that + * an empty packet is a blank password, thus the check for auth_response.empty() has to be made too. + * https://github.com/mysql/mysql-server/blob/8.0/sql/auth/sql_authentication.cc#L4017 + */ + if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1)) + { + LOG_TRACE(log, "Received nonempty password."); + const auto & unpack_auth_response = *auth_response; + const auto * ciphertext = reinterpret_cast(unpack_auth_response.data()); + + unsigned char plaintext[RSA_size(&private_key)]; + int plaintext_size = RSA_private_decrypt(unpack_auth_response.size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING); + if (plaintext_size == -1) + { + if (!sent_public_key) + LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server."); + throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); + } + + password.resize(plaintext_size); + for (int i = 0; i < plaintext_size; i++) + { + password[i] = plaintext[i] ^ static_cast(scramble[i % scramble.size()]); + } + } + else if (is_secure_connection) + { + password = *auth_response; + } + else + { + LOG_TRACE(log, "Received empty password"); + } + + if (!password.empty() && password.back() == 0) + { + password.pop_back(); + } + + context.setUser(user_name, password, address); +} + +#endif + +} + +} + +} diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h new file mode 100644 index 00000000000..3874655e523 --- /dev/null +++ b/src/Core/MySQL/Authentication.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include + +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#if USE_SSL +# include +# include +#endif + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace Authentication +{ + +class IPlugin +{ +public: + virtual ~IPlugin() = default; + + virtual String getName() = 0; + + virtual String getAuthPluginData() = 0; + + virtual void authenticate( + const String & user_name, std::optional auth_response, Context & context, + std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) = 0; +}; + +/// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html +class Native41 : public IPlugin +{ +public: + Native41(); + + Native41(const String & password, const String & auth_plugin_data); + + String getName() override { return "mysql_native_password"; } + + String getAuthPluginData() override { return scramble; } + + void authenticate( + const String & user_name, std::optional auth_response, Context & context, + std::shared_ptr packet_endpoint, bool /* is_secure_connection */, const Poco::Net::SocketAddress & address) override; + +private: + String scramble; +}; + +#if USE_SSL +/// Caching SHA2 plugin is not used because it would be possible to authenticate knowing hash from users.xml. +/// https://dev.mysql.com/doc/internals/en/sha256.html +class Sha256Password : public IPlugin +{ +public: + Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logger * log_); + + String getName() override { return "sha256_password"; } + + String getAuthPluginData() override { return scramble; } + + void authenticate( + const String & user_name, std::optional auth_response, Context & context, + std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) override; + +private: + RSA & public_key; + RSA & private_key; + Poco::Logger * log; + String scramble; +}; +#endif + +} + +} + +} diff --git a/src/Core/MySQL/IMySQLReadPacket.cpp b/src/Core/MySQL/IMySQLReadPacket.cpp new file mode 100644 index 00000000000..8fc8855c8a4 --- /dev/null +++ b/src/Core/MySQL/IMySQLReadPacket.cpp @@ -0,0 +1,81 @@ +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_PACKET_FROM_CLIENT; +} + +namespace MySQLProtocol +{ + +void IMySQLReadPacket::readPayload(ReadBuffer & in, uint8_t & sequence_id) +{ + MySQLPacketPayloadReadBuffer payload(in, sequence_id); + payload.next(); + readPayloadImpl(payload); + if (!payload.eof()) + { + std::stringstream tmp; + tmp << "Packet payload is not fully read. Stopped after " << payload.count() << " bytes, while " << payload.available() << " bytes are in buffer."; + throw Exception(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + } +} + +void IMySQLReadPacket::readPayloadWithUnpacked(ReadBuffer & in) +{ + readPayloadImpl(in); +} + +void LimitedReadPacket::readPayload(ReadBuffer &in, uint8_t &sequence_id) +{ + LimitReadBuffer limited(in, 10000, true, "too long MySQL packet."); + IMySQLReadPacket::readPayload(limited, sequence_id); +} + +void LimitedReadPacket::readPayloadWithUnpacked(ReadBuffer & in) +{ + LimitReadBuffer limited(in, 10000, true, "too long MySQL packet."); + IMySQLReadPacket::readPayloadWithUnpacked(limited); +} + +uint64_t readLengthEncodedNumber(ReadBuffer & buffer) +{ + char c{}; + uint64_t buf = 0; + buffer.readStrict(c); + auto cc = static_cast(c); + if (cc < 0xfc) + { + return cc; + } + else if (cc < 0xfd) + { + buffer.readStrict(reinterpret_cast(&buf), 2); + } + else if (cc < 0xfe) + { + buffer.readStrict(reinterpret_cast(&buf), 3); + } + else + { + buffer.readStrict(reinterpret_cast(&buf), 8); + } + return buf; +} + +void readLengthEncodedString(String & s, ReadBuffer & buffer) +{ + uint64_t len = readLengthEncodedNumber(buffer); + s.resize(len); + buffer.readStrict(reinterpret_cast(s.data()), len); +} + +} + +} diff --git a/src/Core/MySQL/IMySQLReadPacket.h b/src/Core/MySQL/IMySQLReadPacket.h new file mode 100644 index 00000000000..eab31889091 --- /dev/null +++ b/src/Core/MySQL/IMySQLReadPacket.h @@ -0,0 +1,41 @@ +#pragma once + +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +class IMySQLReadPacket +{ +public: + IMySQLReadPacket() = default; + + virtual ~IMySQLReadPacket() = default; + + IMySQLReadPacket(IMySQLReadPacket &&) = default; + + virtual void readPayload(ReadBuffer & in, uint8_t & sequence_id); + + virtual void readPayloadWithUnpacked(ReadBuffer & in); + +protected: + virtual void readPayloadImpl(ReadBuffer & buf) = 0; +}; + +class LimitedReadPacket : public IMySQLReadPacket +{ +public: + void readPayload(ReadBuffer & in, uint8_t & sequence_id) override; + + void readPayloadWithUnpacked(ReadBuffer & in) override; +}; + +uint64_t readLengthEncodedNumber(ReadBuffer & buffer); +void readLengthEncodedString(String & s, ReadBuffer & buffer); + +} + +} diff --git a/src/Core/MySQL/IMySQLWritePacket.cpp b/src/Core/MySQL/IMySQLWritePacket.cpp new file mode 100644 index 00000000000..f5bc339b079 --- /dev/null +++ b/src/Core/MySQL/IMySQLWritePacket.cpp @@ -0,0 +1,86 @@ +#include +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +void IMySQLWritePacket::writePayload(WriteBuffer & buffer, uint8_t & sequence_id) const +{ + MySQLPacketPayloadWriteBuffer buf(buffer, getPayloadSize(), sequence_id); + writePayloadImpl(buf); + buf.next(); + if (buf.remainingPayloadSize()) + { + std::stringstream ss; + ss << "Incomplete payload. Written " << getPayloadSize() - buf.remainingPayloadSize() << " bytes, expected " << getPayloadSize() << " bytes."; + throw Exception(ss.str(), 0); + } +} + +size_t getLengthEncodedNumberSize(uint64_t x) +{ + if (x < 251) + { + return 1; + } + else if (x < (1 << 16)) + { + return 3; + } + else if (x < (1 << 24)) + { + return 4; + } + else + { + return 9; + } +} + +size_t getLengthEncodedStringSize(const String & s) +{ + return getLengthEncodedNumberSize(s.size()) + s.size(); +} + +void writeLengthEncodedNumber(uint64_t x, WriteBuffer & buffer) +{ + if (x < 251) + { + buffer.write(static_cast(x)); + } + else if (x < (1 << 16)) + { + buffer.write(0xfc); + buffer.write(reinterpret_cast(&x), 2); + } + else if (x < (1 << 24)) + { + buffer.write(0xfd); + buffer.write(reinterpret_cast(&x), 3); + } + else + { + buffer.write(0xfe); + buffer.write(reinterpret_cast(&x), 8); + } +} + +void writeLengthEncodedString(const String & s, WriteBuffer & buffer) +{ + writeLengthEncodedNumber(s.size(), buffer); + buffer.write(s.data(), s.size()); +} + +void writeNulTerminatedString(const String & s, WriteBuffer & buffer) +{ + buffer.write(s.data(), s.size()); + buffer.write(0); +} + +} + +} diff --git a/src/Core/MySQL/IMySQLWritePacket.h b/src/Core/MySQL/IMySQLWritePacket.h new file mode 100644 index 00000000000..b00ffbaacae --- /dev/null +++ b/src/Core/MySQL/IMySQLWritePacket.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +class IMySQLWritePacket +{ +public: + IMySQLWritePacket() = default; + + virtual ~IMySQLWritePacket() = default; + + IMySQLWritePacket(IMySQLWritePacket &&) = default; + + virtual void writePayload(WriteBuffer & buffer, uint8_t & sequence_id) const; + +protected: + virtual size_t getPayloadSize() const = 0; + + virtual void writePayloadImpl(WriteBuffer & buffer) const = 0; +}; + +size_t getLengthEncodedNumberSize(uint64_t x); +size_t getLengthEncodedStringSize(const String & s); + +void writeLengthEncodedNumber(uint64_t x, WriteBuffer & buffer); +void writeLengthEncodedString(const String & s, WriteBuffer & buffer); +void writeNulTerminatedString(const String & s, WriteBuffer & buffer); + +} + +} diff --git a/src/Core/MySQLClient.cpp b/src/Core/MySQL/MySQLClient.cpp similarity index 66% rename from src/Core/MySQLClient.cpp rename to src/Core/MySQL/MySQLClient.cpp index 95e83971f2b..71f4e3c01b0 100644 --- a/src/Core/MySQLClient.cpp +++ b/src/Core/MySQL/MySQLClient.cpp @@ -1,10 +1,19 @@ #include "MySQLClient.h" -#include + +#include +#include +#include +#include +#include +#include namespace DB { +using namespace Generic; using namespace Replication; +using namespace ProtocolText; using namespace Authentication; +using namespace ConnectionPhase; namespace ErrorCodes { @@ -44,7 +53,7 @@ void MySQLClient::connect() in = std::make_shared(*socket); out = std::make_shared(*socket); - packet_sender = std::make_shared(*in, *out, seq); + packet_endpoint = std::make_shared(*in, *out, seq); handshake(); } @@ -62,10 +71,10 @@ void MySQLClient::disconnect() void MySQLClient::handshake() { Handshake handshake; - packet_sender->receivePacket(handshake); + packet_endpoint->receivePacket(handshake); if (handshake.auth_plugin_name != mysql_native_password) { - throw MySQLClientError( + throw Exception( "Only support " + mysql_native_password + " auth plugin name, but got " + handshake.auth_plugin_name, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); } @@ -74,48 +83,48 @@ void MySQLClient::handshake() String auth_plugin_data = native41.getAuthPluginData(); HandshakeResponse handshake_response( - client_capability_flags, max_packet_size, charset_utf8, user, "", auth_plugin_data, mysql_native_password); - packet_sender->sendPacket(handshake_response, true); + client_capability_flags, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password); + packet_endpoint->sendPacket(handshake_response, true); - PacketResponse packet_response(client_capability_flags, true); - packet_sender->receivePacket(packet_response); - packet_sender->resetSequenceId(); + ResponsePacket packet_response(client_capability_flags, true); + packet_endpoint->receivePacket(packet_response); + packet_endpoint->resetSequenceId(); if (packet_response.getType() == PACKET_ERR) - throw MySQLClientError(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); else if (packet_response.getType() == PACKET_AUTH_SWITCH) - throw MySQLClientError("Access denied for user " + user, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception("Access denied for user " + user, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); } void MySQLClient::writeCommand(char command, String query) { WriteCommand write_command(command, query); - packet_sender->sendPacket(write_command, true); + packet_endpoint->sendPacket(write_command, true); - PacketResponse packet_response(client_capability_flags); - packet_sender->receivePacket(packet_response); + ResponsePacket packet_response(client_capability_flags); + packet_endpoint->receivePacket(packet_response); switch (packet_response.getType()) { case PACKET_ERR: - throw MySQLClientError(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); case PACKET_OK: break; default: break; } - packet_sender->resetSequenceId(); + packet_endpoint->resetSequenceId(); } void MySQLClient::registerSlaveOnMaster(UInt32 slave_id) { RegisterSlave register_slave(slave_id); - packet_sender->sendPacket(register_slave, true); + packet_endpoint->sendPacket(register_slave, true); - PacketResponse packet_response(client_capability_flags); - packet_sender->receivePacket(packet_response); - packet_sender->resetSequenceId(); + ResponsePacket packet_response(client_capability_flags); + packet_endpoint->receivePacket(packet_response); + packet_endpoint->resetSequenceId(); if (packet_response.getType() == PACKET_ERR) - throw MySQLClientError(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); } void MySQLClient::ping() @@ -141,12 +150,12 @@ void MySQLClient::startBinlogDump(UInt32 slave_id, String replicate_db, String b binlog_pos = binlog_pos < 4 ? 4 : binlog_pos; BinlogDump binlog_dump(binlog_pos, binlog_file_name, slave_id); - packet_sender->sendPacket(binlog_dump, true); + packet_endpoint->sendPacket(binlog_dump, true); } BinlogEventPtr MySQLClient::readOneBinlogEvent(UInt64 milliseconds) { - if (packet_sender->tryReceivePacket(replication, milliseconds)) + if (packet_endpoint->tryReceivePacket(replication, milliseconds)) return replication.readOneEvent(); return {}; diff --git a/src/Core/MySQLClient.h b/src/Core/MySQL/MySQLClient.h similarity index 86% rename from src/Core/MySQLClient.h rename to src/Core/MySQL/MySQLClient.h index ff0b1fbd3a5..708c6b7b3ee 100644 --- a/src/Core/MySQLClient.h +++ b/src/Core/MySQL/MySQLClient.h @@ -1,7 +1,6 @@ #pragma once -#include -#include #include +#include #include #include #include @@ -11,19 +10,15 @@ #include #include #include +#include namespace DB { + using namespace MySQLProtocol; using namespace MySQLReplication; -class MySQLClientError : public DB::Exception -{ -public: - using Exception::Exception; -}; - class MySQLClient { public: @@ -49,7 +44,6 @@ private: uint8_t seq = 0; const UInt8 charset_utf8 = 33; - const UInt32 max_packet_size = MySQLProtocol::MAX_PACKET_LENGTH; const String mysql_native_password = "mysql_native_password"; MySQLFlavor replication; @@ -57,14 +51,14 @@ private: std::shared_ptr out; std::unique_ptr socket; std::optional address; - std::shared_ptr packet_sender; + std::shared_ptr packet_endpoint; void handshake(); void registerSlaveOnMaster(UInt32 slave_id); void writeCommand(char command, String query); }; -class WriteCommand : public WritePacket +class WriteCommand : public IMySQLWritePacket { public: char command; diff --git a/src/Core/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp similarity index 97% rename from src/Core/MySQLReplication.cpp rename to src/Core/MySQL/MySQLReplication.cpp index db0c70d07a1..8d67c91c749 100644 --- a/src/Core/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace DB { @@ -15,6 +17,8 @@ namespace ErrorCodes namespace MySQLReplication { using namespace MySQLProtocol; + using namespace MySQLProtocol::Generic; + using namespace MySQLProtocol::ProtocolText; /// https://dev.mysql.com/doc/internals/en/binlog-event-header.html void EventHeader::parse(ReadBuffer & payload) @@ -59,7 +63,7 @@ namespace MySQLReplication out << "Binlog Version: " << this->binlog_version << std::endl; out << "Server Version: " << this->server_version << std::endl; out << "Create Timestamp: " << this->create_timestamp << std::endl; - out << "Event Header Len: " << this->event_header_length << std::endl; + out << "Event Header Len: " << std::to_string(this->event_header_length) << std::endl; } /// https://dev.mysql.com/doc/internals/en/rotate-event.html @@ -119,7 +123,7 @@ namespace MySQLReplication header.dump(out); out << "Thread ID: " << this->thread_id << std::endl; out << "Execution Time: " << this->exec_time << std::endl; - out << "Schema Len: " << this->schema_len << std::endl; + out << "Schema Len: " << std::to_string(this->schema_len) << std::endl; out << "Error Code: " << this->error_code << std::endl; out << "Status Len: " << this->status_len << std::endl; out << "Schema: " << this->schema << std::endl; @@ -239,14 +243,14 @@ namespace MySQLReplication header.dump(out); out << "Table ID: " << this->table_id << std::endl; out << "Flags: " << this->flags << std::endl; - out << "Schema Len: " << this->schema_len << std::endl; + out << "Schema Len: " << std::to_string(this->schema_len) << std::endl; out << "Schema: " << this->schema << std::endl; - out << "Table Len: " << this->table_len << std::endl; + out << "Table Len: " << std::to_string(this->table_len) << std::endl; out << "Table: " << this->table << std::endl; out << "Column Count: " << this->column_count << std::endl; for (auto i = 0U; i < column_count; i++) { - out << "Column Type [" << i << "]: " << column_type[i] << ", Meta: " << column_meta[i] << std::endl; + out << "Column Type [" << i << "]: " << std::to_string(column_type[i]) << ", Meta: " << column_meta[i] << std::endl; } out << "Null Bitmap: " << this->null_bitmap << std::endl; } @@ -717,8 +721,8 @@ namespace MySQLReplication case PACKET_EOF: throw ReplicationError("Master maybe lost", ErrorCodes::UNKNOWN_EXCEPTION); case PACKET_ERR: - ERR_Packet err; - err.readPayloadImpl(payload); + ERRPacket err; + err.readPayloadWithUnpacked(payload); throw ReplicationError(err.error_message, ErrorCodes::UNKNOWN_EXCEPTION); } // skip the header flag. diff --git a/src/Core/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h similarity index 99% rename from src/Core/MySQLReplication.h rename to src/Core/MySQL/MySQLReplication.h index 29af875a430..444292f346b 100644 --- a/src/Core/MySQLReplication.h +++ b/src/Core/MySQL/MySQLReplication.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include #include #include @@ -13,6 +13,7 @@ namespace DB { + namespace MySQLReplication { static const int EVENT_VERSION_V4 = 4; @@ -465,7 +466,7 @@ namespace MySQLReplication void updateLogName(String binlog) { binlog_name = std::move(binlog); } }; - class IFlavor : public MySQLProtocol::ReadPacket + class IFlavor : public MySQLProtocol::IMySQLReadPacket { public: virtual String getName() const = 0; diff --git a/src/Core/MySQL/PacketEndpoint.cpp b/src/Core/MySQL/PacketEndpoint.cpp new file mode 100644 index 00000000000..0bc5c585516 --- /dev/null +++ b/src/Core/MySQL/PacketEndpoint.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace MySQLProtocol +{ + +PacketEndpoint::PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_) + : sequence_id(sequence_id_), in(nullptr), out(&out_) +{ +} + +PacketEndpoint::PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_) + : sequence_id(sequence_id_), in(&in_), out(&out_) +{ +} + +MySQLPacketPayloadReadBuffer PacketEndpoint::getPayload() +{ + return MySQLPacketPayloadReadBuffer(*in, sequence_id); +} + +void PacketEndpoint::receivePacket(IMySQLReadPacket & packet) +{ + packet.readPayload(*in, sequence_id); +} + +bool PacketEndpoint::tryReceivePacket(IMySQLReadPacket & packet, UInt64 millisecond) +{ + if (millisecond != 0) + { + ReadBufferFromPocoSocket * socket_in = typeid_cast(in); + + if (!socket_in) + throw Exception("LOGICAL ERROR: Attempt to pull the duration in a non socket stream", ErrorCodes::LOGICAL_ERROR); + + if (!socket_in->poll(millisecond * 1000)) + return false; + } + + packet.readPayload(*in, sequence_id); + return true; +} + +void PacketEndpoint::resetSequenceId() +{ + sequence_id = 0; +} + +String PacketEndpoint::packetToText(const String & payload) +{ + String result; + for (auto c : payload) + { + result += ' '; + result += std::to_string(static_cast(c)); + } + return result; +} + +} + +} diff --git a/src/Core/MySQL/PacketEndpoint.h b/src/Core/MySQL/PacketEndpoint.h new file mode 100644 index 00000000000..d027934eafb --- /dev/null +++ b/src/Core/MySQL/PacketEndpoint.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include "IMySQLReadPacket.h" +#include "IMySQLWritePacket.h" +#include "IO/MySQLPacketPayloadReadBuffer.h" + +namespace DB +{ + +namespace MySQLProtocol +{ + +/* Writes and reads packets, keeping sequence-id. + * Throws ProtocolError, if packet with incorrect sequence-id was received. + */ +class PacketEndpoint +{ +public: + uint8_t & sequence_id; + ReadBuffer * in; + WriteBuffer * out; + + /// For writing. + PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_); + + /// For reading and writing. + PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_); + + MySQLPacketPayloadReadBuffer getPayload(); + + void receivePacket(IMySQLReadPacket & packet); + + bool tryReceivePacket(IMySQLReadPacket & packet, UInt64 millisecond = 0); + + /// Sets sequence-id to 0. Must be called before each command phase. + void resetSequenceId(); + + template + void sendPacket(const T & packet, bool flush = false) + { + static_assert(std::is_base_of()); + packet.writePayload(*out, sequence_id); + if (flush) + out->next(); + } + + /// Converts packet to text. Is used for debug output. + static String packetToText(const String & payload); +}; + +} + +} diff --git a/src/Core/MySQL/PacketsConnection.cpp b/src/Core/MySQL/PacketsConnection.cpp new file mode 100644 index 00000000000..32a8a9cf8ab --- /dev/null +++ b/src/Core/MySQL/PacketsConnection.cpp @@ -0,0 +1,242 @@ +#include +#include +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +using namespace Generic; + +namespace ConnectionPhase +{ + +static const size_t SCRAMBLE_LENGTH = 20; +static const size_t AUTH_PLUGIN_DATA_PART_1_LENGTH = 8; + +Handshake::Handshake() : connection_id(0x00), capability_flags(0x00), character_set(0x00), status_flags(0x00) +{ +} + +Handshake::Handshake( + uint32_t capability_flags_, uint32_t connection_id_, + String server_version_, String auth_plugin_name_, String auth_plugin_data_, uint8_t charset_) + : protocol_version(0xa), server_version(std::move(server_version_)), connection_id(connection_id_), capability_flags(capability_flags_), + character_set(charset_), status_flags(0), auth_plugin_name(std::move(auth_plugin_name_)), + auth_plugin_data(std::move(auth_plugin_data_)) +{ +} + +size_t Handshake::getPayloadSize() const +{ + return 26 + server_version.size() + auth_plugin_data.size() + auth_plugin_name.size(); +} + +void Handshake::readPayloadImpl(ReadBuffer & payload) +{ + payload.readStrict(reinterpret_cast(&protocol_version), 1); + readNullTerminated(server_version, payload); + payload.readStrict(reinterpret_cast(&connection_id), 4); + + auth_plugin_data.resize(AUTH_PLUGIN_DATA_PART_1_LENGTH); + payload.readStrict(auth_plugin_data.data(), AUTH_PLUGIN_DATA_PART_1_LENGTH); + + payload.ignore(1); + payload.readStrict(reinterpret_cast(&capability_flags), 2); + payload.readStrict(reinterpret_cast(&character_set), 1); + payload.readStrict(reinterpret_cast(&status_flags), 2); + payload.readStrict((reinterpret_cast(&capability_flags)) + 2, 2); + + UInt8 auth_plugin_data_length = 0; + if (capability_flags & Capability::CLIENT_PLUGIN_AUTH) + { + payload.readStrict(reinterpret_cast(&auth_plugin_data_length), 1); + } + else + { + payload.ignore(1); + } + + payload.ignore(10); + if (capability_flags & Capability::CLIENT_SECURE_CONNECTION) + { + UInt8 part2_length = (SCRAMBLE_LENGTH - AUTH_PLUGIN_DATA_PART_1_LENGTH); + auth_plugin_data.resize(SCRAMBLE_LENGTH); + payload.readStrict(auth_plugin_data.data() + AUTH_PLUGIN_DATA_PART_1_LENGTH, part2_length); + payload.ignore(1); + } + + if (capability_flags & Capability::CLIENT_PLUGIN_AUTH) + { + readNullTerminated(auth_plugin_name, payload); + } +} + +void Handshake::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(static_cast(protocol_version)); + writeNulTerminatedString(server_version, buffer); + buffer.write(reinterpret_cast(&connection_id), 4); + writeNulTerminatedString(auth_plugin_data.substr(0, AUTH_PLUGIN_DATA_PART_1_LENGTH), buffer); + buffer.write(reinterpret_cast(&capability_flags), 2); + buffer.write(reinterpret_cast(&character_set), 1); + buffer.write(reinterpret_cast(&status_flags), 2); + buffer.write((reinterpret_cast(&capability_flags)) + 2, 2); + buffer.write(static_cast(auth_plugin_data.size())); + writeChar(0x0, 10, buffer); + writeString(auth_plugin_data.substr(AUTH_PLUGIN_DATA_PART_1_LENGTH, auth_plugin_data.size() - AUTH_PLUGIN_DATA_PART_1_LENGTH), buffer); + writeString(auth_plugin_name, buffer); + writeChar(0x0, 1, buffer); +} + +HandshakeResponse::HandshakeResponse() : capability_flags(0x00), max_packet_size(0x00), character_set(0x00) +{ +} + +HandshakeResponse::HandshakeResponse( + UInt32 capability_flags_, UInt32 max_packet_size_, UInt8 character_set_, const String & username_, const String & database_, + const String & auth_response_, const String & auth_plugin_name_) + : capability_flags(capability_flags_), max_packet_size(max_packet_size_), character_set(character_set_), username(std::move(username_)), + database(std::move(database_)), auth_response(std::move(auth_response_)), auth_plugin_name(std::move(auth_plugin_name_)) +{ +} + +size_t HandshakeResponse::getPayloadSize() const +{ + size_t size = 0; + size += 4 + 4 + 1 + 23; + size += username.size() + 1; + + if (capability_flags & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA) + { + size += getLengthEncodedStringSize(auth_response); + } + else if (capability_flags & CLIENT_SECURE_CONNECTION) + { + size += (1 + auth_response.size()); + } + else + { + size += (auth_response.size() + 1); + } + if (capability_flags & CLIENT_CONNECT_WITH_DB) + { + size += (database.size() + 1); + } + if (capability_flags & CLIENT_PLUGIN_AUTH) + { + size += (auth_plugin_name.size() + 1); + } + return size; +} + +void HandshakeResponse::readPayloadImpl(ReadBuffer & payload) +{ + payload.readStrict(reinterpret_cast(&capability_flags), 4); + payload.readStrict(reinterpret_cast(&max_packet_size), 4); + payload.readStrict(reinterpret_cast(&character_set), 1); + payload.ignore(23); + + readNullTerminated(username, payload); + + if (capability_flags & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA) + { + readLengthEncodedString(auth_response, payload); + } + else if (capability_flags & CLIENT_SECURE_CONNECTION) + { + char len; + payload.readStrict(len); + auth_response.resize(static_cast(len)); + payload.readStrict(auth_response.data(), len); + } + else + { + readNullTerminated(auth_response, payload); + } + + if (capability_flags & CLIENT_CONNECT_WITH_DB) + { + readNullTerminated(database, payload); + } + + if (capability_flags & CLIENT_PLUGIN_AUTH) + { + readNullTerminated(auth_plugin_name, payload); + } +} + +void HandshakeResponse::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(reinterpret_cast(&capability_flags), 4); + buffer.write(reinterpret_cast(&max_packet_size), 4); + buffer.write(reinterpret_cast(&character_set), 1); + writeChar(0x0, 23, buffer); + + writeNulTerminatedString(username, buffer); + if (capability_flags & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA) + { + writeLengthEncodedString(auth_response, buffer); + } + else if (capability_flags & CLIENT_SECURE_CONNECTION) + { + writeChar(auth_response.size(), buffer); + writeString(auth_response.data(), auth_response.size(), buffer); + } + else + { + writeNulTerminatedString(auth_response, buffer); + } + + if (capability_flags & CLIENT_CONNECT_WITH_DB) + { + writeNulTerminatedString(database, buffer); + } + + if (capability_flags & CLIENT_PLUGIN_AUTH) + { + writeNulTerminatedString(auth_plugin_name, buffer); + } +} + +AuthSwitchRequest::AuthSwitchRequest(String plugin_name_, String auth_plugin_data_) + : plugin_name(std::move(plugin_name_)), auth_plugin_data(std::move(auth_plugin_data_)) +{ +} + +size_t AuthSwitchRequest::getPayloadSize() const +{ + return 2 + plugin_name.size() + auth_plugin_data.size(); +} + +void AuthSwitchRequest::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(0xfe); + writeNulTerminatedString(plugin_name, buffer); + writeString(auth_plugin_data, buffer); +} + +void AuthSwitchResponse::readPayloadImpl(ReadBuffer & payload) +{ + readStringUntilEOF(value, payload); +} + +size_t AuthMoreData::getPayloadSize() const +{ + return 1 + data.size(); +} + +void AuthMoreData::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(0x01); + writeString(data, buffer); +} + +} + +} + +} diff --git a/src/Core/MySQL/PacketsConnection.h b/src/Core/MySQL/PacketsConnection.h new file mode 100644 index 00000000000..30a4beb48f2 --- /dev/null +++ b/src/Core/MySQL/PacketsConnection.h @@ -0,0 +1,110 @@ +#pragma once + +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace ConnectionPhase +{ + +class Handshake : public IMySQLWritePacket, public IMySQLReadPacket +{ +public: + int protocol_version = 0xa; + String server_version; + uint32_t connection_id; + uint32_t capability_flags; + uint8_t character_set; + uint32_t status_flags; + String auth_plugin_name; + String auth_plugin_data; + +protected: + size_t getPayloadSize() const override; + + void readPayloadImpl(ReadBuffer & payload) override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + Handshake(); + + Handshake( + uint32_t capability_flags_, uint32_t connection_id_, + String server_version_, String auth_plugin_name_, String auth_plugin_data_, uint8_t charset_); +}; + +class HandshakeResponse : public IMySQLWritePacket, public IMySQLReadPacket +{ +public: + uint32_t capability_flags; + uint32_t max_packet_size; + uint8_t character_set; + String username; + String database; + String auth_response; + String auth_plugin_name; + +protected: + size_t getPayloadSize() const override; + + void readPayloadImpl(ReadBuffer & payload) override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + HandshakeResponse(); + + HandshakeResponse( + UInt32 capability_flags_, UInt32 max_packet_size_, UInt8 character_set_, + const String & username_, const String & database_, const String & auth_response_, const String & auth_plugin_name_); +}; + +class AuthSwitchRequest : public IMySQLWritePacket +{ +public: + String plugin_name; + String auth_plugin_data; + +protected: + size_t getPayloadSize() const override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + AuthSwitchRequest(String plugin_name_, String auth_plugin_data_); +}; + +class AuthSwitchResponse : public LimitedReadPacket +{ +public: + String value; + +protected: + void readPayloadImpl(ReadBuffer & payload) override; +}; + +class AuthMoreData : public IMySQLWritePacket +{ +public: + String data; + +protected: + size_t getPayloadSize() const override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + explicit AuthMoreData(String data_): data(std::move(data_)) {} +}; + +} + +} + +} diff --git a/src/Core/MySQL/PacketsGeneric.cpp b/src/Core/MySQL/PacketsGeneric.cpp new file mode 100644 index 00000000000..af80797d5c1 --- /dev/null +++ b/src/Core/MySQL/PacketsGeneric.cpp @@ -0,0 +1,262 @@ +#include +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace Generic +{ + +static const size_t MYSQL_ERRMSG_SIZE = 512; + +void SSLRequest::readPayloadImpl(ReadBuffer & buf) +{ + buf.readStrict(reinterpret_cast(&capability_flags), 4); + buf.readStrict(reinterpret_cast(&max_packet_size), 4); + buf.readStrict(reinterpret_cast(&character_set), 1); +} + +OKPacket::OKPacket(uint32_t capabilities_) + : header(0x00), capabilities(capabilities_), affected_rows(0x00), last_insert_id(0x00), status_flags(0x00) +{ +} + +OKPacket::OKPacket( + uint8_t header_, uint32_t capabilities_, uint64_t affected_rows_, uint32_t status_flags_, int16_t warnings_, + String session_state_changes_, String info_) + : header(header_), capabilities(capabilities_), affected_rows(affected_rows_), last_insert_id(0), warnings(warnings_), + status_flags(status_flags_), session_state_changes(std::move(session_state_changes_)), info(std::move(info_)) +{ +} + +size_t OKPacket::getPayloadSize() const +{ + size_t result = 2 + getLengthEncodedNumberSize(affected_rows); + + if (capabilities & CLIENT_PROTOCOL_41) + { + result += 4; + } + else if (capabilities & CLIENT_TRANSACTIONS) + { + result += 2; + } + + if (capabilities & CLIENT_SESSION_TRACK) + { + result += getLengthEncodedStringSize(info); + if (status_flags & SERVER_SESSION_STATE_CHANGED) + result += getLengthEncodedStringSize(session_state_changes); + } + else + { + result += info.size(); + } + + return result; +} + +void OKPacket::readPayloadImpl(ReadBuffer & payload) + +{ + payload.readStrict(reinterpret_cast(&header), 1); + affected_rows = readLengthEncodedNumber(payload); + last_insert_id = readLengthEncodedNumber(payload); + + if (capabilities & CLIENT_PROTOCOL_41) + { + payload.readStrict(reinterpret_cast(&status_flags), 2); + payload.readStrict(reinterpret_cast(&warnings), 2); + } + else if (capabilities & CLIENT_TRANSACTIONS) + { + payload.readStrict(reinterpret_cast(&status_flags), 2); + } + + if (capabilities & CLIENT_SESSION_TRACK) + { + readLengthEncodedString(info, payload); + if (status_flags & SERVER_SESSION_STATE_CHANGED) + { + readLengthEncodedString(session_state_changes, payload); + } + } + else + { + readString(info, payload); + } +} + +void OKPacket::writePayloadImpl(WriteBuffer & buffer) const + +{ + buffer.write(header); + writeLengthEncodedNumber(affected_rows, buffer); + writeLengthEncodedNumber(last_insert_id, buffer); /// last insert-id + + if (capabilities & CLIENT_PROTOCOL_41) + { + buffer.write(reinterpret_cast(&status_flags), 2); + buffer.write(reinterpret_cast(&warnings), 2); + } + else if (capabilities & CLIENT_TRANSACTIONS) + { + buffer.write(reinterpret_cast(&status_flags), 2); + } + + if (capabilities & CLIENT_SESSION_TRACK) + { + writeLengthEncodedString(info, buffer); + if (status_flags & SERVER_SESSION_STATE_CHANGED) + writeLengthEncodedString(session_state_changes, buffer); + } + else + { + writeString(info, buffer); + } +} + +EOFPacket::EOFPacket() : warnings(0x00), status_flags(0x00) +{ +} + +EOFPacket::EOFPacket(int warnings_, int status_flags_) + : warnings(warnings_), status_flags(status_flags_) +{ +} + +size_t EOFPacket::getPayloadSize() const +{ + return 5; +} + +void EOFPacket::readPayloadImpl(ReadBuffer & payload) +{ + payload.readStrict(reinterpret_cast(&header), 1); + assert(header == 0xfe); + payload.readStrict(reinterpret_cast(&warnings), 2); + payload.readStrict(reinterpret_cast(&status_flags), 2); +} + +void EOFPacket::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(header); // EOF header + buffer.write(reinterpret_cast(&warnings), 2); + buffer.write(reinterpret_cast(&status_flags), 2); +} + +void AuthSwitchPacket::readPayloadImpl(ReadBuffer & payload) +{ + payload.readStrict(reinterpret_cast(&header), 1); + assert(header == 0xfe); + readStringUntilEOF(plugin_name, payload); +} + +ERRPacket::ERRPacket() : error_code(0x00) +{ +} + +ERRPacket::ERRPacket(int error_code_, String sql_state_, String error_message_) + : error_code(error_code_), sql_state(std::move(sql_state_)), error_message(std::move(error_message_)) +{ +} + +size_t ERRPacket::getPayloadSize() const +{ + return 4 + sql_state.length() + std::min(error_message.length(), MYSQL_ERRMSG_SIZE); +} + +void ERRPacket::readPayloadImpl(ReadBuffer & payload) +{ + payload.readStrict(reinterpret_cast(&header), 1); + assert(header == 0xff); + + payload.readStrict(reinterpret_cast(&error_code), 2); + + /// SQL State [optional: # + 5bytes string] + UInt8 sharp = static_cast(*payload.position()); + if (sharp == 0x23) + { + payload.ignore(1); + sql_state.resize(5); + payload.readStrict(reinterpret_cast(sql_state.data()), 5); + } + readString(error_message, payload); +} + +void ERRPacket::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(header); + buffer.write(reinterpret_cast(&error_code), 2); + buffer.write('#'); + buffer.write(sql_state.data(), sql_state.length()); + buffer.write(error_message.data(), std::min(error_message.length(), MYSQL_ERRMSG_SIZE)); +} + +ResponsePacket::ResponsePacket(UInt32 server_capability_flags_) + : ok(OKPacket(server_capability_flags_)) +{ +} + +ResponsePacket::ResponsePacket(UInt32 server_capability_flags_, bool is_handshake_) + : ok(OKPacket(server_capability_flags_)), is_handshake(is_handshake_) +{ +} + +void ResponsePacket::readPayloadImpl(ReadBuffer & payload) +{ + UInt16 header = static_cast(*payload.position()); + switch (header) + { + case PACKET_OK: + packetType = PACKET_OK; + ok.readPayloadWithUnpacked(payload); + break; + case PACKET_ERR: + packetType = PACKET_ERR; + err.readPayloadWithUnpacked(payload); + break; + case PACKET_EOF: + if (is_handshake) + { + packetType = PACKET_AUTH_SWITCH; + auth_switch.readPayloadWithUnpacked(payload); + } + else + { + packetType = PACKET_EOF; + eof.readPayloadWithUnpacked(payload); + } + break; + case PACKET_LOCALINFILE: + packetType = PACKET_LOCALINFILE; + break; + default: + packetType = PACKET_OK; + column_length = readLengthEncodedNumber(payload); + } +} + +LengthEncodedNumber::LengthEncodedNumber(uint64_t value_) : value(value_) +{ +} + +size_t LengthEncodedNumber::getPayloadSize() const +{ + return getLengthEncodedNumberSize(value); +} + +void LengthEncodedNumber::writePayloadImpl(WriteBuffer & buffer) const +{ + writeLengthEncodedNumber(value, buffer); +} + +} + +} + +} diff --git a/src/Core/MySQL/PacketsGeneric.h b/src/Core/MySQL/PacketsGeneric.h new file mode 100644 index 00000000000..aec3589ece6 --- /dev/null +++ b/src/Core/MySQL/PacketsGeneric.h @@ -0,0 +1,175 @@ +#pragma once + +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace Generic +{ + +const size_t MAX_PACKET_LENGTH = (1 << 24) - 1; // 16 mb + +enum StatusFlags +{ + SERVER_SESSION_STATE_CHANGED = 0x4000 +}; + +enum Capability +{ + CLIENT_CONNECT_WITH_DB = 0x00000008, + CLIENT_PROTOCOL_41 = 0x00000200, + CLIENT_SSL = 0x00000800, + CLIENT_TRANSACTIONS = 0x00002000, // TODO + CLIENT_SESSION_TRACK = 0x00800000, // TODO + CLIENT_SECURE_CONNECTION = 0x00008000, + CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA = 0x00200000, + CLIENT_PLUGIN_AUTH = 0x00080000, + CLIENT_DEPRECATE_EOF = 0x01000000, +}; + +class SSLRequest : public IMySQLReadPacket +{ +public: + uint32_t capability_flags; + uint32_t max_packet_size; + uint8_t character_set; + +protected: + void readPayloadImpl(ReadBuffer & buf) override; +}; + +class OKPacket : public IMySQLWritePacket, public IMySQLReadPacket +{ +public: + uint8_t header; + uint32_t capabilities; + uint64_t affected_rows; + uint64_t last_insert_id; + int16_t warnings = 0; + uint32_t status_flags; + String session_state_changes; + String info; + +protected: + size_t getPayloadSize() const override; + + void readPayloadImpl(ReadBuffer & payload) override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + OKPacket(uint32_t capabilities_); + + OKPacket(uint8_t header_, uint32_t capabilities_, uint64_t affected_rows_, + uint32_t status_flags_, int16_t warnings_, String session_state_changes_ = "", String info_ = ""); +}; + +class EOFPacket : public IMySQLWritePacket, public IMySQLReadPacket +{ +public: + UInt8 header = 0xfe; + int warnings; + int status_flags; + +protected: + size_t getPayloadSize() const override; + + void readPayloadImpl(ReadBuffer & payload) override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + EOFPacket(); + + EOFPacket(int warnings_, int status_flags_); +}; + +class ERRPacket : public IMySQLWritePacket, public IMySQLReadPacket +{ +public: + UInt8 header = 0xff; + int error_code; + String sql_state; + String error_message; + +protected: + size_t getPayloadSize() const override; + + void readPayloadImpl(ReadBuffer & payload) override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + ERRPacket(); + + ERRPacket(int error_code_, String sql_state_, String error_message_); +}; + +class AuthSwitchPacket : public IMySQLReadPacket +{ +public: + String plugin_name; + + AuthSwitchPacket() = default; + +protected: + UInt8 header = 0x00; + + void readPayloadImpl(ReadBuffer & payload) override; +}; + +enum ResponsePacketType +{ + PACKET_OK = 0x00, + PACKET_ERR = 0xff, + PACKET_EOF = 0xfe, + PACKET_AUTH_SWITCH = 0xfe, + PACKET_LOCALINFILE = 0xfb, +}; + +/// https://dev.mysql.com/doc/internals/en/generic-response-packets.html +class ResponsePacket : public IMySQLReadPacket +{ +public: + OKPacket ok; + ERRPacket err; + EOFPacket eof; + AuthSwitchPacket auth_switch; + UInt64 column_length = 0; + + ResponsePacketType getType() { return packetType; } +protected: + bool is_handshake = false; + ResponsePacketType packetType = PACKET_OK; + + void readPayloadImpl(ReadBuffer & payload) override; + +public: + ResponsePacket(UInt32 server_capability_flags_); + + ResponsePacket(UInt32 server_capability_flags_, bool is_handshake_); +}; + +class LengthEncodedNumber : public IMySQLWritePacket +{ +protected: + uint64_t value; + + size_t getPayloadSize() const override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + explicit LengthEncodedNumber(uint64_t value_); +}; + +} + +} + +} diff --git a/src/Core/MySQL/PacketsProtocolText.cpp b/src/Core/MySQL/PacketsProtocolText.cpp new file mode 100644 index 00000000000..766bcf636e4 --- /dev/null +++ b/src/Core/MySQL/PacketsProtocolText.cpp @@ -0,0 +1,192 @@ +#include +#include +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace ProtocolText +{ + +ResultSetRow::ResultSetRow(const DataTypes & data_types, const Columns & columns_, int row_num_) + : columns(columns_), row_num(row_num_) +{ + for (size_t i = 0; i < columns.size(); i++) + { + if (columns[i]->isNullAt(row_num)) + { + payload_size += 1; + serialized.emplace_back("\xfb"); + } + else + { + WriteBufferFromOwnString ostr; + data_types[i]->serializeAsText(*columns[i], row_num, ostr, FormatSettings()); + payload_size += getLengthEncodedStringSize(ostr.str()); + serialized.push_back(std::move(ostr.str())); + } + } +} + +size_t ResultSetRow::getPayloadSize() const +{ + return payload_size; +} + +void ResultSetRow::writePayloadImpl(WriteBuffer & buffer) const +{ + for (size_t i = 0; i < columns.size(); i++) + { + if (columns[i]->isNullAt(row_num)) + buffer.write(serialized[i].data(), 1); + else + writeLengthEncodedString(serialized[i], buffer); + } +} + +void ComFieldList::readPayloadImpl(ReadBuffer & payload) +{ + // Command byte has been already read from payload. + readNullTerminated(table, payload); + readStringUntilEOF(field_wildcard, payload); +} + +ColumnDefinition::ColumnDefinition() + : character_set(0x00), column_length(0), column_type(MYSQL_TYPE_DECIMAL), flags(0x00) +{ +} + +ColumnDefinition::ColumnDefinition( + String schema_, String table_, String org_table_, String name_, String org_name_, uint16_t character_set_, uint32_t column_length_, + ColumnType column_type_, uint16_t flags_, uint8_t decimals_) + : schema(std::move(schema_)), table(std::move(table_)), org_table(std::move(org_table_)), name(std::move(name_)), + org_name(std::move(org_name_)), character_set(character_set_), column_length(column_length_), column_type(column_type_), + flags(flags_), decimals(decimals_) +{ +} + +ColumnDefinition::ColumnDefinition( + String name_, uint16_t character_set_, uint32_t column_length_, ColumnType column_type_, uint16_t flags_, uint8_t decimals_) + : ColumnDefinition("", "", "", std::move(name_), "", character_set_, column_length_, column_type_, flags_, decimals_) +{ +} + +size_t ColumnDefinition::getPayloadSize() const +{ + return 13 + getLengthEncodedStringSize("def") + getLengthEncodedStringSize(schema) + getLengthEncodedStringSize(table) + getLengthEncodedStringSize(org_table) + \ + getLengthEncodedStringSize(name) + getLengthEncodedStringSize(org_name) + getLengthEncodedNumberSize(next_length); +} + +void ColumnDefinition::readPayloadImpl(ReadBuffer & payload) +{ + String def; + readLengthEncodedString(def, payload); + assert(def == "def"); + readLengthEncodedString(schema, payload); + readLengthEncodedString(table, payload); + readLengthEncodedString(org_table, payload); + readLengthEncodedString(name, payload); + readLengthEncodedString(org_name, payload); + next_length = readLengthEncodedNumber(payload); + payload.readStrict(reinterpret_cast(&character_set), 2); + payload.readStrict(reinterpret_cast(&column_length), 4); + payload.readStrict(reinterpret_cast(&column_type), 1); + payload.readStrict(reinterpret_cast(&flags), 2); + payload.readStrict(reinterpret_cast(&decimals), 2); + payload.ignore(2); +} + +void ColumnDefinition::writePayloadImpl(WriteBuffer & buffer) const +{ + writeLengthEncodedString(std::string("def"), buffer); /// always "def" + writeLengthEncodedString(schema, buffer); + writeLengthEncodedString(table, buffer); + writeLengthEncodedString(org_table, buffer); + writeLengthEncodedString(name, buffer); + writeLengthEncodedString(org_name, buffer); + writeLengthEncodedNumber(next_length, buffer); + buffer.write(reinterpret_cast(&character_set), 2); + buffer.write(reinterpret_cast(&column_length), 4); + buffer.write(reinterpret_cast(&column_type), 1); + buffer.write(reinterpret_cast(&flags), 2); + buffer.write(reinterpret_cast(&decimals), 2); + writeChar(0x0, 2, buffer); +} + +ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex type_index) +{ + ColumnType column_type; + CharacterSet charset = CharacterSet::binary; + int flags = 0; + switch (type_index) + { + case TypeIndex::UInt8: + column_type = ColumnType::MYSQL_TYPE_TINY; + flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; + break; + case TypeIndex::UInt16: + column_type = ColumnType::MYSQL_TYPE_SHORT; + flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; + break; + case TypeIndex::UInt32: + column_type = ColumnType::MYSQL_TYPE_LONG; + flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; + break; + case TypeIndex::UInt64: + column_type = ColumnType::MYSQL_TYPE_LONGLONG; + flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; + break; + case TypeIndex::Int8: + column_type = ColumnType::MYSQL_TYPE_TINY; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::Int16: + column_type = ColumnType::MYSQL_TYPE_SHORT; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::Int32: + column_type = ColumnType::MYSQL_TYPE_LONG; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::Int64: + column_type = ColumnType::MYSQL_TYPE_LONGLONG; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::Float32: + column_type = ColumnType::MYSQL_TYPE_FLOAT; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::Float64: + column_type = ColumnType::MYSQL_TYPE_DOUBLE; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::Date: + column_type = ColumnType::MYSQL_TYPE_DATE; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::DateTime: + column_type = ColumnType::MYSQL_TYPE_DATETIME; + flags = ColumnDefinitionFlags::BINARY_FLAG; + break; + case TypeIndex::String: + case TypeIndex::FixedString: + column_type = ColumnType::MYSQL_TYPE_STRING; + charset = CharacterSet::utf8_general_ci; + break; + default: + column_type = ColumnType::MYSQL_TYPE_STRING; + charset = CharacterSet::utf8_general_ci; + break; + } + return ColumnDefinition(column_name, charset, 0, column_type, flags, 0); +} + +} + +} + +} diff --git a/src/Core/MySQL/PacketsProtocolText.h b/src/Core/MySQL/PacketsProtocolText.h new file mode 100644 index 00000000000..94c2308c8e9 --- /dev/null +++ b/src/Core/MySQL/PacketsProtocolText.h @@ -0,0 +1,157 @@ +#pragma once + +#include +#include + +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace ProtocolText +{ + +enum CharacterSet +{ + utf8_general_ci = 33, + binary = 63 +}; + +// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__column__definition__flags.html +enum ColumnDefinitionFlags +{ + UNSIGNED_FLAG = 32, + BINARY_FLAG = 128 +}; + +enum ColumnType +{ + MYSQL_TYPE_DECIMAL = 0x00, + MYSQL_TYPE_TINY = 0x01, + MYSQL_TYPE_SHORT = 0x02, + MYSQL_TYPE_LONG = 0x03, + MYSQL_TYPE_FLOAT = 0x04, + MYSQL_TYPE_DOUBLE = 0x05, + MYSQL_TYPE_NULL = 0x06, + MYSQL_TYPE_TIMESTAMP = 0x07, + MYSQL_TYPE_LONGLONG = 0x08, + MYSQL_TYPE_INT24 = 0x09, + MYSQL_TYPE_DATE = 0x0a, + MYSQL_TYPE_TIME = 0x0b, + MYSQL_TYPE_DATETIME = 0x0c, + MYSQL_TYPE_YEAR = 0x0d, + MYSQL_TYPE_NEWDATE = 0x0e, + MYSQL_TYPE_VARCHAR = 0x0f, + MYSQL_TYPE_BIT = 0x10, + MYSQL_TYPE_TIMESTAMP2 = 0x11, + MYSQL_TYPE_DATETIME2 = 0x12, + MYSQL_TYPE_TIME2 = 0x13, + MYSQL_TYPE_JSON = 0xf5, + MYSQL_TYPE_NEWDECIMAL = 0xf6, + MYSQL_TYPE_ENUM = 0xf7, + MYSQL_TYPE_SET = 0xf8, + MYSQL_TYPE_TINY_BLOB = 0xf9, + MYSQL_TYPE_MEDIUM_BLOB = 0xfa, + MYSQL_TYPE_LONG_BLOB = 0xfb, + MYSQL_TYPE_BLOB = 0xfc, + MYSQL_TYPE_VAR_STRING = 0xfd, + MYSQL_TYPE_STRING = 0xfe, + MYSQL_TYPE_GEOMETRY = 0xff +}; + +enum Command +{ + COM_SLEEP = 0x0, + COM_QUIT = 0x1, + COM_INIT_DB = 0x2, + COM_QUERY = 0x3, + COM_FIELD_LIST = 0x4, + COM_CREATE_DB = 0x5, + COM_DROP_DB = 0x6, + COM_REFRESH = 0x7, + COM_SHUTDOWN = 0x8, + COM_STATISTICS = 0x9, + COM_PROCESS_INFO = 0xa, + COM_CONNECT = 0xb, + COM_PROCESS_KILL = 0xc, + COM_DEBUG = 0xd, + COM_PING = 0xe, + COM_TIME = 0xf, + COM_DELAYED_INSERT = 0x10, + COM_CHANGE_USER = 0x11, + COM_BINLOG_DUMP = 0x12, + COM_REGISTER_SLAVE = 0x15, + COM_RESET_CONNECTION = 0x1f, + COM_DAEMON = 0x1d +}; + +class ResultSetRow : public IMySQLWritePacket +{ +protected: + const Columns & columns; + int row_num; + size_t payload_size = 0; + std::vector serialized; + + size_t getPayloadSize() const override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + ResultSetRow(const DataTypes & data_types, const Columns & columns_, int row_num_); +}; + +class ComFieldList : public LimitedReadPacket +{ +public: + String table, field_wildcard; + + void readPayloadImpl(ReadBuffer & payload) override; +}; + +class ColumnDefinition : public IMySQLWritePacket, public IMySQLReadPacket +{ +public: + String schema; + String table; + String org_table; + String name; + String org_name; + size_t next_length = 0x0c; + uint16_t character_set; + uint32_t column_length; + ColumnType column_type; + uint16_t flags; + uint8_t decimals = 0x00; + +protected: + size_t getPayloadSize() const override; + + void readPayloadImpl(ReadBuffer & payload) override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + ColumnDefinition(); + + ColumnDefinition( + String schema_, String table_, String org_table_, String name_, String org_name_, uint16_t character_set_, uint32_t column_length_, + ColumnType column_type_, uint16_t flags_, uint8_t decimals_); + + /// Should be used when column metadata (original name, table, original table, database) is unknown. + ColumnDefinition( + String name_, uint16_t character_set_, uint32_t column_length_, ColumnType column_type_, uint16_t flags_, uint8_t decimals_); + +}; + +ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex index); + +} + +} + +} diff --git a/src/Core/MySQL/PacketsReplication.cpp b/src/Core/MySQL/PacketsReplication.cpp new file mode 100644 index 00000000000..b1e9efaa9ad --- /dev/null +++ b/src/Core/MySQL/PacketsReplication.cpp @@ -0,0 +1,62 @@ +#include + +#include +#include +#include + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace Replication +{ + +RegisterSlave::RegisterSlave(UInt32 server_id_) + : server_id(server_id_), slaves_mysql_port(0x00), replication_rank(0x00), master_id(0x00) +{ +} + +size_t RegisterSlave::getPayloadSize() const +{ + return 1 + 4 + getLengthEncodedStringSize(slaves_hostname) + getLengthEncodedStringSize(slaves_users) + + getLengthEncodedStringSize(slaves_password) + 2 + 4 + 4; +} + +void RegisterSlave::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(ProtocolText::COM_REGISTER_SLAVE); + buffer.write(reinterpret_cast(&server_id), 4); + writeLengthEncodedString(slaves_hostname, buffer); + writeLengthEncodedString(slaves_users, buffer); + writeLengthEncodedString(slaves_password, buffer); + buffer.write(reinterpret_cast(&slaves_mysql_port), 2); + buffer.write(reinterpret_cast(&replication_rank), 4); + buffer.write(reinterpret_cast(&master_id), 4); +} + +BinlogDump::BinlogDump(UInt32 binlog_pos_, String binlog_file_name_, UInt32 server_id_) + : binlog_pos(binlog_pos_), flags(0x00), server_id(server_id_), binlog_file_name(std::move(binlog_file_name_)) +{ +} + +size_t BinlogDump::getPayloadSize() const +{ + return 1 + 4 + 2 + 4 + binlog_file_name.size() + 1; +} + +void BinlogDump::writePayloadImpl(WriteBuffer & buffer) const +{ + buffer.write(ProtocolText::COM_BINLOG_DUMP); + buffer.write(reinterpret_cast(&binlog_pos), 4); + buffer.write(reinterpret_cast(&flags), 2); + buffer.write(reinterpret_cast(&server_id), 4); + buffer.write(binlog_file_name.data(), binlog_file_name.length()); + buffer.write(0x00); +} + +} + +} +} diff --git a/src/Core/MySQL/PacketsReplication.h b/src/Core/MySQL/PacketsReplication.h new file mode 100644 index 00000000000..908301a91b7 --- /dev/null +++ b/src/Core/MySQL/PacketsReplication.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include + +/// Implementation of MySQL wire protocol. +/// Works only on little-endian architecture. + +namespace DB +{ + +namespace MySQLProtocol +{ + +namespace Replication +{ + +/// https://dev.mysql.com/doc/internals/en/com-register-slave.html +class RegisterSlave : public IMySQLWritePacket +{ +public: + UInt32 server_id; + String slaves_hostname; + String slaves_users; + String slaves_password; + size_t slaves_mysql_port; + UInt32 replication_rank; + UInt32 master_id; + +protected: + size_t getPayloadSize() const override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + RegisterSlave(UInt32 server_id_); +}; + +/// https://dev.mysql.com/doc/internals/en/com-binlog-dump.html +class BinlogDump : public IMySQLWritePacket +{ +public: + UInt32 binlog_pos; + UInt16 flags; + UInt32 server_id; + String binlog_file_name; + +protected: + size_t getPayloadSize() const override; + + void writePayloadImpl(WriteBuffer & buffer) const override; + +public: + BinlogDump(UInt32 binlog_pos_, String binlog_file_name_, UInt32 server_id_); +}; + +} +} + +} diff --git a/src/Core/MySQLProtocol.cpp b/src/Core/MySQLProtocol.cpp deleted file mode 100644 index c303101d7cc..00000000000 --- a/src/Core/MySQLProtocol.cpp +++ /dev/null @@ -1,171 +0,0 @@ -#include "MySQLProtocol.h" -#include -#include -#include -#include - -#include -#include - - -namespace DB::MySQLProtocol -{ - -void PacketSender::resetSequenceId() -{ - sequence_id = 0; -} - -String PacketSender::packetToText(const String & payload) -{ - String result; - for (auto c : payload) - { - result += ' '; - result += std::to_string(static_cast(c)); - } - return result; -} - -uint64_t readLengthEncodedNumber(ReadBuffer & ss) -{ - char c{}; - uint64_t buf = 0; - ss.readStrict(c); - auto cc = static_cast(c); - if (cc < 0xfc) - { - return cc; - } - else if (cc < 0xfd) - { - ss.readStrict(reinterpret_cast(&buf), 2); - } - else if (cc < 0xfe) - { - ss.readStrict(reinterpret_cast(&buf), 3); - } - else - { - ss.readStrict(reinterpret_cast(&buf), 8); - } - return buf; -} - -void writeLengthEncodedNumber(uint64_t x, WriteBuffer & buffer) -{ - if (x < 251) - { - buffer.write(static_cast(x)); - } - else if (x < (1 << 16)) - { - buffer.write(0xfc); - buffer.write(reinterpret_cast(&x), 2); - } - else if (x < (1 << 24)) - { - buffer.write(0xfd); - buffer.write(reinterpret_cast(&x), 3); - } - else - { - buffer.write(0xfe); - buffer.write(reinterpret_cast(&x), 8); - } -} - -size_t getLengthEncodedNumberSize(uint64_t x) -{ - if (x < 251) - { - return 1; - } - else if (x < (1 << 16)) - { - return 3; - } - else if (x < (1 << 24)) - { - return 4; - } - else - { - return 9; - } -} - -size_t getLengthEncodedStringSize(const String & s) -{ - return getLengthEncodedNumberSize(s.size()) + s.size(); -} - -ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex type_index) -{ - ColumnType column_type; - CharacterSet charset = CharacterSet::binary; - int flags = 0; - switch (type_index) - { - case TypeIndex::UInt8: - column_type = ColumnType::MYSQL_TYPE_TINY; - flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; - break; - case TypeIndex::UInt16: - column_type = ColumnType::MYSQL_TYPE_SHORT; - flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; - break; - case TypeIndex::UInt32: - column_type = ColumnType::MYSQL_TYPE_LONG; - flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; - break; - case TypeIndex::UInt64: - column_type = ColumnType::MYSQL_TYPE_LONGLONG; - flags = ColumnDefinitionFlags::BINARY_FLAG | ColumnDefinitionFlags::UNSIGNED_FLAG; - break; - case TypeIndex::Int8: - column_type = ColumnType::MYSQL_TYPE_TINY; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::Int16: - column_type = ColumnType::MYSQL_TYPE_SHORT; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::Int32: - column_type = ColumnType::MYSQL_TYPE_LONG; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::Int64: - column_type = ColumnType::MYSQL_TYPE_LONGLONG; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::Float32: - column_type = ColumnType::MYSQL_TYPE_FLOAT; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::Float64: - column_type = ColumnType::MYSQL_TYPE_DOUBLE; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::Date: - column_type = ColumnType::MYSQL_TYPE_DATE; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::DateTime: - column_type = ColumnType::MYSQL_TYPE_DATETIME; - flags = ColumnDefinitionFlags::BINARY_FLAG; - break; - case TypeIndex::String: - case TypeIndex::FixedString: - column_type = ColumnType::MYSQL_TYPE_STRING; - charset = CharacterSet::utf8_general_ci; - break; - default: - column_type = ColumnType::MYSQL_TYPE_STRING; - charset = CharacterSet::utf8_general_ci; - break; - } - return ColumnDefinition(column_name, charset, 0, column_type, flags, 0); -} - -} diff --git a/src/Core/MySQLProtocol.h b/src/Core/MySQLProtocol.h deleted file mode 100644 index fb17b48fb0d..00000000000 --- a/src/Core/MySQLProtocol.h +++ /dev/null @@ -1,1563 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif - -#if USE_SSL -# include -# include -#endif - -/// Implementation of MySQL wire protocol. -/// Works only on little-endian architecture. - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; - extern const int CANNOT_WRITE_AFTER_END_OF_BUFFER; - extern const int UNKNOWN_PACKET_FROM_CLIENT; - extern const int MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES; - extern const int OPENSSL_ERROR; - extern const int UNKNOWN_EXCEPTION; -} - -namespace MySQLProtocol -{ - -const size_t MAX_PACKET_LENGTH = (1 << 24) - 1; // 16 mb -const size_t SCRAMBLE_LENGTH = 20; -const size_t AUTH_PLUGIN_DATA_PART_1_LENGTH = 8; -const size_t MYSQL_ERRMSG_SIZE = 512; -const size_t PACKET_HEADER_SIZE = 4; -const size_t SSL_REQUEST_PAYLOAD_SIZE = 32; - - -enum CharacterSet -{ - utf8_general_ci = 33, - binary = 63 -}; - -enum StatusFlags -{ - SERVER_SESSION_STATE_CHANGED = 0x4000 -}; - -enum Capability -{ - CLIENT_CONNECT_WITH_DB = 0x00000008, - CLIENT_PROTOCOL_41 = 0x00000200, - CLIENT_SSL = 0x00000800, - CLIENT_TRANSACTIONS = 0x00002000, // TODO - CLIENT_SESSION_TRACK = 0x00800000, // TODO - CLIENT_SECURE_CONNECTION = 0x00008000, - CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA = 0x00200000, - CLIENT_PLUGIN_AUTH = 0x00080000, - CLIENT_DEPRECATE_EOF = 0x01000000, -}; - -enum Command -{ - COM_SLEEP = 0x0, - COM_QUIT = 0x1, - COM_INIT_DB = 0x2, - COM_QUERY = 0x3, - COM_FIELD_LIST = 0x4, - COM_CREATE_DB = 0x5, - COM_DROP_DB = 0x6, - COM_REFRESH = 0x7, - COM_SHUTDOWN = 0x8, - COM_STATISTICS = 0x9, - COM_PROCESS_INFO = 0xa, - COM_CONNECT = 0xb, - COM_PROCESS_KILL = 0xc, - COM_DEBUG = 0xd, - COM_PING = 0xe, - COM_TIME = 0xf, - COM_DELAYED_INSERT = 0x10, - COM_CHANGE_USER = 0x11, - COM_BINLOG_DUMP = 0x12, - COM_REGISTER_SLAVE = 0x15, - COM_RESET_CONNECTION = 0x1f, - COM_DAEMON = 0x1d -}; - -enum ColumnType -{ - MYSQL_TYPE_DECIMAL = 0x00, - MYSQL_TYPE_TINY = 0x01, - MYSQL_TYPE_SHORT = 0x02, - MYSQL_TYPE_LONG = 0x03, - MYSQL_TYPE_FLOAT = 0x04, - MYSQL_TYPE_DOUBLE = 0x05, - MYSQL_TYPE_NULL = 0x06, - MYSQL_TYPE_TIMESTAMP = 0x07, - MYSQL_TYPE_LONGLONG = 0x08, - MYSQL_TYPE_INT24 = 0x09, - MYSQL_TYPE_DATE = 0x0a, - MYSQL_TYPE_TIME = 0x0b, - MYSQL_TYPE_DATETIME = 0x0c, - MYSQL_TYPE_YEAR = 0x0d, - MYSQL_TYPE_NEWDATE = 0x0e, - MYSQL_TYPE_VARCHAR = 0x0f, - MYSQL_TYPE_BIT = 0x10, - MYSQL_TYPE_TIMESTAMP2 = 0x11, - MYSQL_TYPE_DATETIME2 = 0x12, - MYSQL_TYPE_TIME2 = 0x13, - MYSQL_TYPE_JSON = 0xf5, - MYSQL_TYPE_NEWDECIMAL = 0xf6, - MYSQL_TYPE_ENUM = 0xf7, - MYSQL_TYPE_SET = 0xf8, - MYSQL_TYPE_TINY_BLOB = 0xf9, - MYSQL_TYPE_MEDIUM_BLOB = 0xfa, - MYSQL_TYPE_LONG_BLOB = 0xfb, - MYSQL_TYPE_BLOB = 0xfc, - MYSQL_TYPE_VAR_STRING = 0xfd, - MYSQL_TYPE_STRING = 0xfe, - MYSQL_TYPE_GEOMETRY = 0xff -}; - -enum ResponsePacketType -{ - PACKET_OK = 0x00, - PACKET_ERR = 0xff, - PACKET_EOF = 0xfe, - PACKET_AUTH_SWITCH = 0xfe, - PACKET_LOCALINFILE = 0xfb, -}; - -// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__column__definition__flags.html -enum ColumnDefinitionFlags -{ - UNSIGNED_FLAG = 32, - BINARY_FLAG = 128 -}; - - -class ProtocolError : public DB::Exception -{ -public: - using Exception::Exception; -}; - - -/** Reading packets. - * Internally, it calls (if no more data) next() method of the underlying ReadBufferFromPocoSocket, and sets the working buffer to the rest part of the current packet payload. - */ -class PacketPayloadReadBuffer : public ReadBuffer -{ -public: - PacketPayloadReadBuffer(ReadBuffer & in_, uint8_t & sequence_id_) - : ReadBuffer(in_.position(), 0) // not in.buffer().begin(), because working buffer may include previous packet - , in(in_) - , sequence_id(sequence_id_) - { - } - -private: - ReadBuffer & in; - uint8_t & sequence_id; - const size_t max_packet_size = MAX_PACKET_LENGTH; - - bool has_read_header = false; - - // Size of packet which is being read now. - size_t payload_length = 0; - - // Offset in packet payload. - size_t offset = 0; - -protected: - bool nextImpl() override - { - if (!has_read_header || (payload_length == max_packet_size && offset == payload_length)) - { - has_read_header = true; - working_buffer.resize(0); - offset = 0; - payload_length = 0; - in.readStrict(reinterpret_cast(&payload_length), 3); - - if (payload_length > max_packet_size) - { - std::ostringstream tmp; - tmp << "Received packet with payload larger than max_packet_size: " << payload_length; - throw ProtocolError(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); - } - - size_t packet_sequence_id = 0; - in.read(reinterpret_cast(packet_sequence_id)); - if (packet_sequence_id != sequence_id) - { - std::ostringstream tmp; - tmp << "Received packet with wrong sequence-id: " << packet_sequence_id << ". Expected: " << static_cast(sequence_id) << '.'; - throw ProtocolError(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); - } - sequence_id++; - - if (payload_length == 0) - return false; - } - else if (offset == payload_length) - { - return false; - } - - in.nextIfAtEnd(); - working_buffer = ReadBuffer::Buffer(in.position(), in.buffer().end()); - size_t count = std::min(in.available(), payload_length - offset); - working_buffer.resize(count); - in.ignore(count); - - offset += count; - - return true; - } -}; - - -class ReadPacket -{ -public: - ReadPacket() = default; - - ReadPacket(ReadPacket &&) = default; - - virtual void readPayload(ReadBuffer & in, uint8_t & sequence_id) - { - PacketPayloadReadBuffer payload(in, sequence_id); - payload.next(); - readPayloadImpl(payload); - if (!payload.eof()) - { - std::stringstream tmp; - tmp << "Packet payload is not fully read. Stopped after " << payload.count() << " bytes, while " << payload.available() << " bytes are in buffer."; - throw ProtocolError(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); - } - } - - virtual void readPayloadImpl(ReadBuffer & buf) = 0; - - virtual ~ReadPacket() = default; -}; - - -class LimitedReadPacket : public ReadPacket -{ -public: - void readPayload(ReadBuffer & in, uint8_t & sequence_id) override - { - LimitReadBuffer limited(in, 10000, true, "too long MySQL packet."); - ReadPacket::readPayload(limited, sequence_id); - } -}; - - -/** Writing packets. - * https://dev.mysql.com/doc/internals/en/mysql-packet.html - */ -class PacketPayloadWriteBuffer : public WriteBuffer -{ -public: - PacketPayloadWriteBuffer(WriteBuffer & out_, size_t payload_length_, uint8_t & sequence_id_) - : WriteBuffer(out_.position(), 0), out(out_), sequence_id(sequence_id_), total_left(payload_length_) - { - startNewPacket(); - setWorkingBuffer(); - pos = out.position(); - } - - bool remainingPayloadSize() - { - return total_left; - } - -private: - WriteBuffer & out; - uint8_t & sequence_id; - - size_t total_left = 0; - size_t payload_length = 0; - size_t bytes_written = 0; - bool eof = false; - - void startNewPacket() - { - payload_length = std::min(total_left, MAX_PACKET_LENGTH); - bytes_written = 0; - total_left -= payload_length; - - out.write(reinterpret_cast(&payload_length), 3); - out.write(sequence_id++); - bytes += 4; - } - - /// Sets working buffer to the rest of current packet payload. - void setWorkingBuffer() - { - out.nextIfAtEnd(); - working_buffer = WriteBuffer::Buffer(out.position(), out.position() + std::min(payload_length - bytes_written, out.available())); - - if (payload_length - bytes_written == 0) - { - /// Finished writing packet. Due to an implementation of WriteBuffer, working_buffer cannot be empty. Further write attempts will throw Exception. - eof = true; - working_buffer.resize(1); - } - } - -protected: - void nextImpl() override - { - const int written = pos - working_buffer.begin(); - if (eof) - throw Exception("Cannot write after end of buffer.", ErrorCodes::CANNOT_WRITE_AFTER_END_OF_BUFFER); - - out.position() += written; - bytes_written += written; - - /// Packets of size greater than MAX_PACKET_LENGTH are split into few packets of size MAX_PACKET_LENGTH and las packet of size < MAX_PACKET_LENGTH. - if (bytes_written == payload_length && (total_left > 0 || payload_length == MAX_PACKET_LENGTH)) - startNewPacket(); - - setWorkingBuffer(); - } -}; - - -class WritePacket -{ -public: - virtual void writePayload(WriteBuffer & buffer, uint8_t & sequence_id) const - { - PacketPayloadWriteBuffer buf(buffer, getPayloadSize(), sequence_id); - writePayloadImpl(buf); - buf.next(); - if (buf.remainingPayloadSize()) - { - std::stringstream ss; - ss << "Incomplete payload. Written " << getPayloadSize() - buf.remainingPayloadSize() << " bytes, expected " << getPayloadSize() << " bytes."; - throw Exception(ss.str(), 0); - } - } - - virtual ~WritePacket() = default; - -protected: - virtual size_t getPayloadSize() const = 0; - - virtual void writePayloadImpl(WriteBuffer & buffer) const = 0; -}; - -/* Writes and reads packets, keeping sequence-id. - * Throws ProtocolError, if packet with incorrect sequence-id was received. - */ -class PacketSender -{ -public: - uint8_t & sequence_id; - ReadBuffer * in; - WriteBuffer * out; - size_t max_packet_size = MAX_PACKET_LENGTH; - - /// For reading and writing. - PacketSender(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_) - : sequence_id(sequence_id_) - , in(&in_) - , out(&out_) - { - } - - /// For writing. - PacketSender(WriteBuffer & out_, uint8_t & sequence_id_) - : sequence_id(sequence_id_) - , in(nullptr) - , out(&out_) - { - } - - void receivePacket(ReadPacket & packet) - { - packet.readPayload(*in, sequence_id); - } - - bool tryReceivePacket(ReadPacket & packet, UInt64 millisecond = 0) - { - if (millisecond != 0) - { - ReadBufferFromPocoSocket * socket_in = typeid_cast(in); - - if (!socket_in) - throw Exception("LOGICAL ERROR: Attempt to pull the duration in a non socket stream", ErrorCodes::LOGICAL_ERROR); - - if (!socket_in->poll(millisecond * 1000)) - return false; - } - - packet.readPayload(*in, sequence_id); - return true; - } - - template - void sendPacket(const T & packet, bool flush = false) - { - static_assert(std::is_base_of()); - packet.writePayload(*out, sequence_id); - if (flush) - out->next(); - } - - PacketPayloadReadBuffer getPayload() - { - return PacketPayloadReadBuffer(*in, sequence_id); - } - - /// Sets sequence-id to 0. Must be called before each command phase. - void resetSequenceId(); - - /// Converts packet to text. Is used for debug output. - static String packetToText(const String & payload); -}; - - -uint64_t readLengthEncodedNumber(ReadBuffer & ss); - -inline void readLengthEncodedString(String & s, ReadBuffer & buffer) -{ - uint64_t len = readLengthEncodedNumber(buffer); - s.resize(len); - buffer.readStrict(reinterpret_cast(s.data()), len); -} - -void writeLengthEncodedNumber(uint64_t x, WriteBuffer & buffer); - -inline void writeLengthEncodedString(const String & s, WriteBuffer & buffer) -{ - writeLengthEncodedNumber(s.size(), buffer); - buffer.write(s.data(), s.size()); -} - -inline void writeNulTerminatedString(const String & s, WriteBuffer & buffer) -{ - buffer.write(s.data(), s.size()); - buffer.write(0); -} - -size_t getLengthEncodedNumberSize(uint64_t x); - -size_t getLengthEncodedStringSize(const String & s); - - -class Handshake : public WritePacket, public ReadPacket -{ -public: - int protocol_version = 0xa; - String server_version; - uint32_t connection_id; - uint32_t capability_flags; - uint8_t character_set; - uint32_t status_flags; - String auth_plugin_name; - String auth_plugin_data; - - Handshake() : connection_id(0x00), capability_flags(0x00), character_set(0x00), status_flags(0x00) { } - - Handshake(uint32_t capability_flags_, uint32_t connection_id_, String server_version_, String auth_plugin_name_, String auth_plugin_data_) - : protocol_version(0xa) - , server_version(std::move(server_version_)) - , connection_id(connection_id_) - , capability_flags(capability_flags_) - , character_set(CharacterSet::utf8_general_ci) - , status_flags(0) - , auth_plugin_name(std::move(auth_plugin_name_)) - , auth_plugin_data(std::move(auth_plugin_data_)) - { - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(static_cast(protocol_version)); - writeNulTerminatedString(server_version, buffer); - buffer.write(reinterpret_cast(&connection_id), 4); - writeNulTerminatedString(auth_plugin_data.substr(0, AUTH_PLUGIN_DATA_PART_1_LENGTH), buffer); - buffer.write(reinterpret_cast(&capability_flags), 2); - buffer.write(reinterpret_cast(&character_set), 1); - buffer.write(reinterpret_cast(&status_flags), 2); - buffer.write((reinterpret_cast(&capability_flags)) + 2, 2); - buffer.write(static_cast(auth_plugin_data.size())); - writeChar(0x0, 10, buffer); - writeString(auth_plugin_data.substr(AUTH_PLUGIN_DATA_PART_1_LENGTH, auth_plugin_data.size() - AUTH_PLUGIN_DATA_PART_1_LENGTH), buffer); - writeString(auth_plugin_name, buffer); - writeChar(0x0, 1, buffer); - } - - void readPayloadImpl(ReadBuffer & payload) override - { - payload.readStrict(reinterpret_cast(&protocol_version), 1); - readNullTerminated(server_version, payload); - payload.readStrict(reinterpret_cast(&connection_id), 4); - - auth_plugin_data.resize(AUTH_PLUGIN_DATA_PART_1_LENGTH); - payload.readStrict(auth_plugin_data.data(), AUTH_PLUGIN_DATA_PART_1_LENGTH); - - payload.ignore(1); - payload.readStrict(reinterpret_cast(&capability_flags), 2); - payload.readStrict(reinterpret_cast(&character_set), 1); - payload.readStrict(reinterpret_cast(&status_flags), 2); - payload.readStrict((reinterpret_cast(&capability_flags)) + 2, 2); - - UInt8 auth_plugin_data_length = 0; - if (capability_flags & MySQLProtocol::CLIENT_PLUGIN_AUTH) - { - payload.readStrict(reinterpret_cast(&auth_plugin_data_length), 1); - } - else - { - payload.ignore(1); - } - - payload.ignore(10); - if (capability_flags & MySQLProtocol::CLIENT_SECURE_CONNECTION) - { - UInt8 part2_length = (SCRAMBLE_LENGTH - AUTH_PLUGIN_DATA_PART_1_LENGTH); - auth_plugin_data.resize(SCRAMBLE_LENGTH); - payload.readStrict(auth_plugin_data.data() + AUTH_PLUGIN_DATA_PART_1_LENGTH, part2_length); - payload.ignore(1); - } - - if (capability_flags & MySQLProtocol::CLIENT_PLUGIN_AUTH) - { - readNullTerminated(auth_plugin_name, payload); - } - } - -protected: - size_t getPayloadSize() const override - { - return 26 + server_version.size() + auth_plugin_data.size() + auth_plugin_name.size(); - } -}; - -class SSLRequest : public ReadPacket -{ -public: - uint32_t capability_flags; - uint32_t max_packet_size; - uint8_t character_set; - - void readPayloadImpl(ReadBuffer & buf) override - { - buf.readStrict(reinterpret_cast(&capability_flags), 4); - buf.readStrict(reinterpret_cast(&max_packet_size), 4); - buf.readStrict(reinterpret_cast(&character_set), 1); - } -}; - -class HandshakeResponse : public WritePacket, public ReadPacket -{ -public: - uint32_t capability_flags; - uint32_t max_packet_size; - uint8_t character_set; - String username; - String database; - String auth_response; - String auth_plugin_name; - - HandshakeResponse() : capability_flags(0x00), max_packet_size(0x00), character_set(0x00) { } - - HandshakeResponse( - UInt32 capability_flags_, - UInt32 max_packet_size_, - UInt8 character_set_, - const String & username_, - const String & database_, - const String & auth_response_, - const String & auth_plugin_name_) - : capability_flags(capability_flags_) - , max_packet_size(max_packet_size_) - , character_set(character_set_) - , username(std::move(username_)) - , database(std::move(database_)) - , auth_response(std::move(auth_response_)) - , auth_plugin_name(std::move(auth_plugin_name_)) - { - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(reinterpret_cast(&capability_flags), 4); - buffer.write(reinterpret_cast(&max_packet_size), 4); - buffer.write(reinterpret_cast(&character_set), 1); - writeChar(0x0, 23, buffer); - - writeNulTerminatedString(username, buffer); - if (capability_flags & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA) - { - writeLengthEncodedString(auth_response, buffer); - } - else if (capability_flags & CLIENT_SECURE_CONNECTION) - { - writeChar(auth_response.size(), buffer); - writeString(auth_response.data(), auth_response.size(), buffer); - } - else - { - writeNulTerminatedString(auth_response, buffer); - } - - if (capability_flags & CLIENT_CONNECT_WITH_DB) - { - writeNulTerminatedString(database, buffer); - } - - if (capability_flags & CLIENT_PLUGIN_AUTH) - { - writeNulTerminatedString(auth_plugin_name, buffer); - } - } - - void readPayloadImpl(ReadBuffer & payload) override - { - payload.readStrict(reinterpret_cast(&capability_flags), 4); - payload.readStrict(reinterpret_cast(&max_packet_size), 4); - payload.readStrict(reinterpret_cast(&character_set), 1); - payload.ignore(23); - - readNullTerminated(username, payload); - - if (capability_flags & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA) - { - readLengthEncodedString(auth_response, payload); - } - else if (capability_flags & CLIENT_SECURE_CONNECTION) - { - char len; - payload.readStrict(len); - auth_response.resize(static_cast(len)); - payload.readStrict(auth_response.data(), len); - } - else - { - readNullTerminated(auth_response, payload); - } - - if (capability_flags & CLIENT_CONNECT_WITH_DB) - { - readNullTerminated(database, payload); - } - - if (capability_flags & CLIENT_PLUGIN_AUTH) - { - readNullTerminated(auth_plugin_name, payload); - } - } - -protected: - size_t getPayloadSize() const override - { - size_t size = 0; - size += 4 + 4 + 1 + 23; - size += username.size() + 1; - - if (capability_flags & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA) - { - size += getLengthEncodedStringSize(auth_response); - } - else if (capability_flags & CLIENT_SECURE_CONNECTION) - { - size += (1 + auth_response.size()); - } - else - { - size += (auth_response.size() + 1); - } - if (capability_flags & CLIENT_CONNECT_WITH_DB) - { - size += (database.size() + 1); - } - if (capability_flags & CLIENT_PLUGIN_AUTH) - { - size += (auth_plugin_name.size() + 1); - } - return size; - } -}; - -class AuthSwitchRequest : public WritePacket -{ - String plugin_name; - String auth_plugin_data; -public: - AuthSwitchRequest(String plugin_name_, String auth_plugin_data_) - : plugin_name(std::move(plugin_name_)), auth_plugin_data(std::move(auth_plugin_data_)) - { - } - -protected: - size_t getPayloadSize() const override - { - return 2 + plugin_name.size() + auth_plugin_data.size(); - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(0xfe); - writeNulTerminatedString(plugin_name, buffer); - writeString(auth_plugin_data, buffer); - } -}; - -class AuthSwitchResponse : public LimitedReadPacket -{ -public: - String value; - - void readPayloadImpl(ReadBuffer & payload) override - { - readStringUntilEOF(value, payload); - } -}; - -class AuthMoreData : public WritePacket -{ - String data; -public: - explicit AuthMoreData(String data_): data(std::move(data_)) {} - -protected: - size_t getPayloadSize() const override - { - return 1 + data.size(); - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(0x01); - writeString(data, buffer); - } -}; - - -class OK_Packet : public WritePacket, public ReadPacket -{ -public: - uint8_t header; - uint32_t capabilities; - uint64_t affected_rows; - uint64_t last_insert_id; - int16_t warnings = 0; - uint32_t status_flags; - String session_state_changes; - String info; - - OK_Packet(uint32_t capabilities_) - : header(0x00), capabilities(capabilities_), affected_rows(0x00), last_insert_id(0x00), status_flags(0x00) - { - } - - OK_Packet( - uint8_t header_, - uint32_t capabilities_, - uint64_t affected_rows_, - uint32_t status_flags_, - int16_t warnings_, - String session_state_changes_ = "", - String info_ = "") - : header(header_) - , capabilities(capabilities_) - , affected_rows(affected_rows_) - , last_insert_id(0) - , warnings(warnings_) - , status_flags(status_flags_) - , session_state_changes(std::move(session_state_changes_)) - , info(std::move(info_)) - { - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(header); - writeLengthEncodedNumber(affected_rows, buffer); - writeLengthEncodedNumber(last_insert_id, buffer); /// last insert-id - - if (capabilities & CLIENT_PROTOCOL_41) - { - buffer.write(reinterpret_cast(&status_flags), 2); - buffer.write(reinterpret_cast(&warnings), 2); - } - else if (capabilities & CLIENT_TRANSACTIONS) - { - buffer.write(reinterpret_cast(&status_flags), 2); - } - - if (capabilities & CLIENT_SESSION_TRACK) - { - writeLengthEncodedString(info, buffer); - if (status_flags & SERVER_SESSION_STATE_CHANGED) - writeLengthEncodedString(session_state_changes, buffer); - } - else - { - writeString(info, buffer); - } - } - - void readPayloadImpl(ReadBuffer & payload) override - { - payload.readStrict(reinterpret_cast(&header), 1); - affected_rows = readLengthEncodedNumber(payload); - last_insert_id = readLengthEncodedNumber(payload); - - if (capabilities & CLIENT_PROTOCOL_41) - { - payload.readStrict(reinterpret_cast(&status_flags), 2); - payload.readStrict(reinterpret_cast(&warnings), 2); - } - else if (capabilities & CLIENT_TRANSACTIONS) - { - payload.readStrict(reinterpret_cast(&status_flags), 2); - } - - if (capabilities & CLIENT_SESSION_TRACK) - { - readLengthEncodedString(info, payload); - if (status_flags & SERVER_SESSION_STATE_CHANGED) - { - readLengthEncodedString(session_state_changes, payload); - } - } - else - { - readString(info, payload); - } - } - -protected: - size_t getPayloadSize() const override - { - size_t result = 2 + getLengthEncodedNumberSize(affected_rows); - - if (capabilities & CLIENT_PROTOCOL_41) - { - result += 4; - } - else if (capabilities & CLIENT_TRANSACTIONS) - { - result += 2; - } - - if (capabilities & CLIENT_SESSION_TRACK) - { - result += getLengthEncodedStringSize(info); - if (status_flags & SERVER_SESSION_STATE_CHANGED) - result += getLengthEncodedStringSize(session_state_changes); - } - else - { - result += info.size(); - } - - return result; - } -}; - -class EOF_Packet : public WritePacket, public ReadPacket -{ -public: - UInt8 header = 0xfe; - int warnings; - int status_flags; - - EOF_Packet() : warnings(0x00), status_flags(0x00) { } - - EOF_Packet(int warnings_, int status_flags_) : warnings(warnings_), status_flags(status_flags_) { } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(header); // EOF header - buffer.write(reinterpret_cast(&warnings), 2); - buffer.write(reinterpret_cast(&status_flags), 2); - } - - void readPayloadImpl(ReadBuffer & payload) override - { - payload.readStrict(reinterpret_cast(&header), 1); - assert(header == 0xfe); - payload.readStrict(reinterpret_cast(&warnings), 2); - payload.readStrict(reinterpret_cast(&status_flags), 2); - } - -protected: - size_t getPayloadSize() const override - { - return 5; - } -}; - -class AuthSwitch_Packet : public ReadPacket -{ -public: - String plugin_name; - - AuthSwitch_Packet() { } - - void readPayloadImpl(ReadBuffer & payload) override - { - payload.readStrict(reinterpret_cast(&header), 1); - assert(header == 0xfe); - readStringUntilEOF(plugin_name, payload); - } - -private: - UInt8 header = 0x00; -}; - -class ERR_Packet : public WritePacket, public ReadPacket -{ -public: - UInt8 header = 0xff; - int error_code; - String sql_state; - String error_message; - - ERR_Packet() : error_code(0x00) { } - - ERR_Packet(int error_code_, String sql_state_, String error_message_) - : error_code(error_code_), sql_state(std::move(sql_state_)), error_message(std::move(error_message_)) - { - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(header); - buffer.write(reinterpret_cast(&error_code), 2); - buffer.write('#'); - buffer.write(sql_state.data(), sql_state.length()); - buffer.write(error_message.data(), std::min(error_message.length(), MYSQL_ERRMSG_SIZE)); - } - - void readPayloadImpl(ReadBuffer & payload) override - { - payload.readStrict(reinterpret_cast(&header), 1); - assert(header == 0xff); - - payload.readStrict(reinterpret_cast(&error_code), 2); - - /// SQL State [optional: # + 5bytes string] - UInt8 sharp = static_cast(*payload.position()); - if (sharp == 0x23) - { - payload.ignore(1); - sql_state.resize(5); - payload.readStrict(reinterpret_cast(sql_state.data()), 5); - } - readString(error_message, payload); - } - -protected: - size_t getPayloadSize() const override - { - return 4 + sql_state.length() + std::min(error_message.length(), MYSQL_ERRMSG_SIZE); - } -}; - -/// https://dev.mysql.com/doc/internals/en/generic-response-packets.html -class PacketResponse : public ReadPacket -{ -public: - OK_Packet ok; - ERR_Packet err; - EOF_Packet eof; - AuthSwitch_Packet auth_switch; - UInt64 column_length = 0; - - PacketResponse(UInt32 server_capability_flags_) : ok(OK_Packet(server_capability_flags_)) { } - PacketResponse(UInt32 server_capability_flags_, bool is_handshake_) - : ok(OK_Packet(server_capability_flags_)), is_handshake(is_handshake_) - { - } - - void readPayloadImpl(ReadBuffer & payload) override - { - UInt16 header = static_cast(*payload.position()); - switch (header) - { - case PACKET_OK: - packetType = PACKET_OK; - ok.readPayloadImpl(payload); - break; - case PACKET_ERR: - packetType = PACKET_ERR; - err.readPayloadImpl(payload); - break; - case PACKET_EOF: - if (is_handshake) - { - packetType = PACKET_AUTH_SWITCH; - auth_switch.readPayloadImpl(payload); - } - else - { - packetType = PACKET_EOF; - eof.readPayloadImpl(payload); - } - break; - case PACKET_LOCALINFILE: - packetType = PACKET_LOCALINFILE; - break; - default: - packetType = PACKET_OK; - column_length = readLengthEncodedNumber(payload); - } - } - - ResponsePacketType getType() { return packetType; } - -private: - bool is_handshake = false; - ResponsePacketType packetType = PACKET_OK; -}; - -class ColumnDefinition : public WritePacket, public ReadPacket -{ -public: - String schema; - String table; - String org_table; - String name; - String org_name; - size_t next_length = 0x0c; - uint16_t character_set; - uint32_t column_length; - ColumnType column_type; - uint16_t flags; - uint8_t decimals = 0x00; - - ColumnDefinition() : character_set(0x00), column_length(0), column_type(MYSQL_TYPE_DECIMAL), flags(0x00) { } - - ColumnDefinition( - String schema_, - String table_, - String org_table_, - String name_, - String org_name_, - uint16_t character_set_, - uint32_t column_length_, - ColumnType column_type_, - uint16_t flags_, - uint8_t decimals_) - - : schema(std::move(schema_)), table(std::move(table_)), org_table(std::move(org_table_)), name(std::move(name_)), - org_name(std::move(org_name_)), character_set(character_set_), column_length(column_length_), column_type(column_type_), flags(flags_), - decimals(decimals_) - { - } - - /// Should be used when column metadata (original name, table, original table, database) is unknown. - ColumnDefinition( - String name_, - uint16_t character_set_, - uint32_t column_length_, - ColumnType column_type_, - uint16_t flags_, - uint8_t decimals_) - : ColumnDefinition("", "", "", std::move(name_), "", character_set_, column_length_, column_type_, flags_, decimals_) - { - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - writeLengthEncodedString(std::string("def"), buffer); /// always "def" - writeLengthEncodedString(schema, buffer); - writeLengthEncodedString(table, buffer); - writeLengthEncodedString(org_table, buffer); - writeLengthEncodedString(name, buffer); - writeLengthEncodedString(org_name, buffer); - writeLengthEncodedNumber(next_length, buffer); - buffer.write(reinterpret_cast(&character_set), 2); - buffer.write(reinterpret_cast(&column_length), 4); - buffer.write(reinterpret_cast(&column_type), 1); - buffer.write(reinterpret_cast(&flags), 2); - buffer.write(reinterpret_cast(&decimals), 2); - writeChar(0x0, 2, buffer); - } - - void readPayloadImpl(ReadBuffer & payload) override - { - String def; - readLengthEncodedString(def, payload); - assert(def == "def"); - readLengthEncodedString(schema, payload); - readLengthEncodedString(table, payload); - readLengthEncodedString(org_table, payload); - readLengthEncodedString(name, payload); - readLengthEncodedString(org_name, payload); - next_length = readLengthEncodedNumber(payload); - payload.readStrict(reinterpret_cast(&character_set), 2); - payload.readStrict(reinterpret_cast(&column_length), 4); - payload.readStrict(reinterpret_cast(&column_type), 1); - payload.readStrict(reinterpret_cast(&flags), 2); - payload.readStrict(reinterpret_cast(&decimals), 2); - payload.ignore(2); - } - -protected: - size_t getPayloadSize() const override - { - return 13 + getLengthEncodedStringSize("def") + getLengthEncodedStringSize(schema) + getLengthEncodedStringSize(table) + getLengthEncodedStringSize(org_table) + \ - getLengthEncodedStringSize(name) + getLengthEncodedStringSize(org_name) + getLengthEncodedNumberSize(next_length); - } -}; - -class ComFieldList : public LimitedReadPacket -{ -public: - String table, field_wildcard; - - void readPayloadImpl(ReadBuffer & payload) override - { - // Command byte has been already read from payload. - readNullTerminated(table, payload); - readStringUntilEOF(field_wildcard, payload); - } -}; - -class LengthEncodedNumber : public WritePacket -{ - uint64_t value; -public: - explicit LengthEncodedNumber(uint64_t value_): value(value_) - { - } - -protected: - size_t getPayloadSize() const override - { - return getLengthEncodedNumberSize(value); - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - writeLengthEncodedNumber(value, buffer); - } -}; - - -ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex index); - - -namespace ProtocolText -{ - -class ResultsetRow : public WritePacket -{ - const Columns & columns; - int row_num; - size_t payload_size = 0; - std::vector serialized; -public: - ResultsetRow(const DataTypes & data_types, const Columns & columns_, int row_num_) - : columns(columns_) - , row_num(row_num_) - { - for (size_t i = 0; i < columns.size(); i++) - { - if (columns[i]->isNullAt(row_num)) - { - payload_size += 1; - serialized.emplace_back("\xfb"); - } - else - { - WriteBufferFromOwnString ostr; - data_types[i]->serializeAsText(*columns[i], row_num, ostr, FormatSettings()); - payload_size += getLengthEncodedStringSize(ostr.str()); - serialized.push_back(std::move(ostr.str())); - } - } - } -protected: - size_t getPayloadSize() const override - { - return payload_size; - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - for (size_t i = 0; i < columns.size(); i++) - { - if (columns[i]->isNullAt(row_num)) - buffer.write(serialized[i].data(), 1); - else - writeLengthEncodedString(serialized[i], buffer); - } - } -}; - -} - -namespace Authentication -{ - -class IPlugin -{ -public: - virtual String getName() = 0; - - virtual String getAuthPluginData() = 0; - - virtual void authenticate(const String & user_name, std::optional auth_response, Context & context, std::shared_ptr packet_sender, bool is_secure_connection, - const Poco::Net::SocketAddress & address) = 0; - - virtual ~IPlugin() = default; -}; - -/// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html -class Native41 : public IPlugin -{ -public: - Native41() - { - scramble.resize(SCRAMBLE_LENGTH + 1, 0); - Poco::RandomInputStream generator; - - /** Generate a random string using ASCII characters but avoid separator character, - * produce pseudo random numbers between with about 7 bit worth of entropty between 1-127. - * https://github.com/mysql/mysql-server/blob/8.0/mysys/crypt_genhash_impl.cc#L427 - */ - for (size_t i = 0; i < SCRAMBLE_LENGTH; ++i) - { - generator >> scramble[i]; - scramble[i] &= 0x7f; - if (scramble[i] == '\0' || scramble[i] == '$') - scramble[i] = scramble[i] + 1; - } - } - - Native41(const String & password, const String & auth_plugin_data) - { - /// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html - /// SHA1( password ) XOR SHA1( "20-bytes random data from server" SHA1( SHA1( password ) ) ) - Poco::SHA1Engine engine1; - engine1.update(password.data()); - const Poco::SHA1Engine::Digest & password_sha1 = engine1.digest(); - - Poco::SHA1Engine engine2; - engine2.update(password_sha1.data(), password_sha1.size()); - const Poco::SHA1Engine::Digest & password_double_sha1 = engine2.digest(); - - Poco::SHA1Engine engine3; - engine3.update(auth_plugin_data.data(), auth_plugin_data.size()); - engine3.update(password_double_sha1.data(), password_double_sha1.size()); - const Poco::SHA1Engine::Digest & digest = engine3.digest(); - - scramble.resize(SCRAMBLE_LENGTH); - for (size_t i = 0; i < SCRAMBLE_LENGTH; i++) - { - scramble[i] = static_cast(password_sha1[i] ^ digest[i]); - } - } - - String getName() override - { - return "mysql_native_password"; - } - - String getAuthPluginData() override - { - return scramble; - } - - void authenticate( - const String & user_name, - std::optional auth_response, - Context & context, - std::shared_ptr packet_sender, - bool /* is_secure_connection */, - const Poco::Net::SocketAddress & address) override - { - if (!auth_response) - { - packet_sender->sendPacket(AuthSwitchRequest(getName(), scramble), true); - AuthSwitchResponse response; - packet_sender->receivePacket(response); - auth_response = response.value; - } - - if (auth_response->empty()) - { - context.setUser(user_name, "", address); - return; - } - - if (auth_response->size() != Poco::SHA1Engine::DIGEST_SIZE) - throw Exception("Wrong size of auth response. Expected: " + std::to_string(Poco::SHA1Engine::DIGEST_SIZE) + " bytes, received: " + std::to_string(auth_response->size()) + " bytes.", - ErrorCodes::UNKNOWN_EXCEPTION); - - auto user = context.getAccessControlManager().read(user_name); - - Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordDoubleSHA1(); - assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE); - - Poco::SHA1Engine engine; - engine.update(scramble.data(), SCRAMBLE_LENGTH); - engine.update(double_sha1_value.data(), double_sha1_value.size()); - - String password_sha1(Poco::SHA1Engine::DIGEST_SIZE, 0x0); - const Poco::SHA1Engine::Digest & digest = engine.digest(); - for (size_t i = 0; i < password_sha1.size(); i++) - { - password_sha1[i] = digest[i] ^ static_cast((*auth_response)[i]); - } - context.setUser(user_name, password_sha1, address); - } -private: - String scramble; -}; - -#if USE_SSL -/// Caching SHA2 plugin is not used because it would be possible to authenticate knowing hash from users.xml. -/// https://dev.mysql.com/doc/internals/en/sha256.html -class Sha256Password : public IPlugin -{ -public: - Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logger * log_) - : public_key(public_key_) - , private_key(private_key_) - , log(log_) - { - /** Native authentication sent 20 bytes + '\0' character = 21 bytes. - * This plugin must do the same to stay consistent with historical behavior if it is set to operate as a default plugin. [1] - * https://github.com/mysql/mysql-server/blob/8.0/sql/auth/sql_authentication.cc#L3994 - */ - scramble.resize(SCRAMBLE_LENGTH + 1, 0); - Poco::RandomInputStream generator; - - for (size_t i = 0; i < SCRAMBLE_LENGTH; ++i) - { - generator >> scramble[i]; - scramble[i] &= 0x7f; - if (scramble[i] == '\0' || scramble[i] == '$') - scramble[i] = scramble[i] + 1; - } - } - - String getName() override - { - return "sha256_password"; - } - - String getAuthPluginData() override - { - return scramble; - } - - void authenticate( - const String & user_name, - std::optional auth_response, - Context & context, - std::shared_ptr packet_sender, - bool is_secure_connection, - const Poco::Net::SocketAddress & address) override - { - if (!auth_response) - { - packet_sender->sendPacket(AuthSwitchRequest(getName(), scramble), true); - - if (packet_sender->in->eof()) - throw Exception("Client doesn't support authentication method " + getName() + " used by ClickHouse. Specifying user password using 'password_double_sha1_hex' may fix the problem.", - ErrorCodes::MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES); - - AuthSwitchResponse response; - packet_sender->receivePacket(response); - auth_response = response.value; - LOG_TRACE(log, "Authentication method mismatch."); - } - else - { - LOG_TRACE(log, "Authentication method match."); - } - - bool sent_public_key = false; - if (auth_response == "\1") - { - LOG_TRACE(log, "Client requests public key."); - BIO * mem = BIO_new(BIO_s_mem()); - SCOPE_EXIT(BIO_free(mem)); - if (PEM_write_bio_RSA_PUBKEY(mem, &public_key) != 1) - { - throw Exception("Failed to write public key to memory. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); - } - char * pem_buf = nullptr; -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wold-style-cast" - long pem_size = BIO_get_mem_data(mem, &pem_buf); -# pragma GCC diagnostic pop - String pem(pem_buf, pem_size); - - LOG_TRACE(log, "Key: {}", pem); - - AuthMoreData data(pem); - packet_sender->sendPacket(data, true); - sent_public_key = true; - - AuthSwitchResponse response; - packet_sender->receivePacket(response); - auth_response = response.value; - } - else - { - LOG_TRACE(log, "Client didn't request public key."); - } - - String password; - - /** Decrypt password, if it's not empty. - * The original intention was that the password is a string[NUL] but this never got enforced properly so now we have to accept that - * an empty packet is a blank password, thus the check for auth_response.empty() has to be made too. - * https://github.com/mysql/mysql-server/blob/8.0/sql/auth/sql_authentication.cc#L4017 - */ - if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1)) - { - LOG_TRACE(log, "Received nonempty password."); - auto ciphertext = reinterpret_cast(auth_response->data()); - - unsigned char plaintext[RSA_size(&private_key)]; - int plaintext_size = RSA_private_decrypt(auth_response->size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING); - if (plaintext_size == -1) - { - if (!sent_public_key) - LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server."); - throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR); - } - - password.resize(plaintext_size); - for (int i = 0; i < plaintext_size; i++) - { - password[i] = plaintext[i] ^ static_cast(scramble[i % scramble.size()]); - } - } - else if (is_secure_connection) - { - password = *auth_response; - } - else - { - LOG_TRACE(log, "Received empty password"); - } - - if (!password.empty() && password.back() == 0) - { - password.pop_back(); - } - - context.setUser(user_name, password, address); - } - -private: - RSA & public_key; - RSA & private_key; - Poco::Logger * log; - String scramble; -}; -#endif - -} - -namespace Replication -{ - /// https://dev.mysql.com/doc/internals/en/com-register-slave.html - class RegisterSlave : public WritePacket - { - public: - UInt8 header = COM_REGISTER_SLAVE; - UInt32 server_id; - String slaves_hostname; - String slaves_users; - String slaves_password; - size_t slaves_mysql_port; - UInt32 replication_rank; - UInt32 master_id; - - RegisterSlave(UInt32 server_id_) : server_id(server_id_), slaves_mysql_port(0x00), replication_rank(0x00), master_id(0x00) { } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(header); - buffer.write(reinterpret_cast(&server_id), 4); - writeLengthEncodedString(slaves_hostname, buffer); - writeLengthEncodedString(slaves_users, buffer); - writeLengthEncodedString(slaves_password, buffer); - buffer.write(reinterpret_cast(&slaves_mysql_port), 2); - buffer.write(reinterpret_cast(&replication_rank), 4); - buffer.write(reinterpret_cast(&master_id), 4); - } - - protected: - size_t getPayloadSize() const override - { - return 1 + 4 + getLengthEncodedStringSize(slaves_hostname) + getLengthEncodedStringSize(slaves_users) - + getLengthEncodedStringSize(slaves_password) + 2 + 4 + 4; - } - }; - - /// https://dev.mysql.com/doc/internals/en/com-binlog-dump.html - class BinlogDump : public WritePacket - { - public: - UInt8 header = COM_BINLOG_DUMP; - UInt32 binlog_pos; - UInt16 flags; - UInt32 server_id; - String binlog_file_name; - - BinlogDump(UInt32 binlog_pos_, String binlog_file_name_, UInt32 server_id_) - : binlog_pos(binlog_pos_), flags(0x00), server_id(server_id_), binlog_file_name(std::move(binlog_file_name_)) - { - } - - void writePayloadImpl(WriteBuffer & buffer) const override - { - buffer.write(header); - buffer.write(reinterpret_cast(&binlog_pos), 4); - buffer.write(reinterpret_cast(&flags), 2); - buffer.write(reinterpret_cast(&server_id), 4); - buffer.write(binlog_file_name.data(), binlog_file_name.length()); - buffer.write(0x00); - } - - protected: - size_t getPayloadSize() const override { return 1 + 4 + 2 + 4 + binlog_file_name.size() + 1; } - }; -} -} - -} diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 60a3792ea87..54775c79b14 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -347,7 +347,6 @@ class IColumn; M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Ordinary, "Default database engine.", 0) \ - M(Bool, allow_experimental_database_atomic, true, "Allow to create database with Engine=Atomic.", 0) \ M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ M(Bool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ @@ -395,7 +394,8 @@ class IColumn; M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ \ M(Bool, force_optimize_skip_unused_shards_no_nested, false, "Obsolete setting, does nothing. Will be removed after 2020-12-01. Use force_optimize_skip_unused_shards_nesting instead.", 0) \ - M(Bool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) + M(Bool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \ + M(Bool, allow_experimental_database_atomic, true, "Obsolete setting, does nothing. Will be removed after 2021-02-12", 0) #define FORMAT_FACTORY_SETTINGS(M) \ M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \ diff --git a/src/Core/tests/mysql_protocol.cpp b/src/Core/tests/mysql_protocol.cpp index e5e875b1975..6e12b6fea4f 100644 --- a/src/Core/tests/mysql_protocol.cpp +++ b/src/Core/tests/mysql_protocol.cpp @@ -1,7 +1,11 @@ #include -#include -#include +#include +#include +#include +#include +#include +#include #include #include @@ -11,15 +15,19 @@ int main(int argc, char ** argv) { using namespace DB; using namespace MySQLProtocol; + using namespace MySQLProtocol::Generic; using namespace MySQLProtocol::Authentication; + using namespace MySQLProtocol::ConnectionPhase; + using namespace MySQLProtocol::ProtocolText; + uint8_t sequence_id = 1; String user = "default"; String password = "123"; String database; UInt8 charset_utf8 = 33; - UInt32 max_packet_size = MySQLProtocol::MAX_PACKET_LENGTH; + UInt32 max_packet_size = MAX_PACKET_LENGTH; String mysql_native_password = "mysql_native_password"; UInt32 server_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH @@ -34,13 +42,13 @@ int main(int argc, char ** argv) std::string s0; WriteBufferFromString out0(s0); - Handshake server_handshake(server_capability_flags, -1, "ClickHouse", "mysql_native_password", "aaaaaaaaaaaaaaaaaaaaa"); - server_handshake.writePayloadImpl(out0); + Handshake server_handshake(server_capability_flags, -1, "ClickHouse", "mysql_native_password", "aaaaaaaaaaaaaaaaaaaaa", CharacterSet::utf8_general_ci); + server_handshake.writePayload(out0, sequence_id); /// 1.2 Client reads the greeting ReadBufferFromString in0(s0); Handshake client_handshake; - client_handshake.readPayloadImpl(in0); + client_handshake.readPayload(in0, sequence_id); /// Check packet ASSERT(server_handshake.capability_flags == client_handshake.capability_flags) @@ -59,12 +67,12 @@ int main(int argc, char ** argv) String auth_plugin_data = native41.getAuthPluginData(); HandshakeResponse client_handshake_response( client_capability_flags, max_packet_size, charset_utf8, user, database, auth_plugin_data, mysql_native_password); - client_handshake_response.writePayloadImpl(out1); + client_handshake_response.writePayload(out1, sequence_id); /// 2.2 Server reads the response ReadBufferFromString in1(s1); HandshakeResponse server_handshake_response; - server_handshake_response.readPayloadImpl(in1); + server_handshake_response.readPayload(in1, sequence_id); /// Check ASSERT(server_handshake_response.capability_flags == client_handshake_response.capability_flags) @@ -80,13 +88,13 @@ int main(int argc, char ** argv) // 1. Server writes packet std::string s0; WriteBufferFromString out0(s0); - OK_Packet server(0x00, server_capability_flags, 0, 0, 0, "", ""); - server.writePayloadImpl(out0); + OKPacket server(0x00, server_capability_flags, 0, 0, 0, "", ""); + server.writePayload(out0, sequence_id); // 2. Client reads packet ReadBufferFromString in0(s0); - PacketResponse client(server_capability_flags); - client.readPayloadImpl(in0); + ResponsePacket client(server_capability_flags); + client.readPayload(in0, sequence_id); // Check ASSERT(client.getType() == PACKET_OK) @@ -100,13 +108,13 @@ int main(int argc, char ** argv) // 1. Server writes packet std::string s0; WriteBufferFromString out0(s0); - ERR_Packet server(123, "12345", "This is the error message"); - server.writePayloadImpl(out0); + ERRPacket server(123, "12345", "This is the error message"); + server.writePayload(out0, sequence_id); // 2. Client reads packet ReadBufferFromString in0(s0); - PacketResponse client(server_capability_flags); - client.readPayloadImpl(in0); + ResponsePacket client(server_capability_flags); + client.readPayload(in0, sequence_id); // Check ASSERT(client.getType() == PACKET_ERR) @@ -121,13 +129,13 @@ int main(int argc, char ** argv) // 1. Server writes packet std::string s0; WriteBufferFromString out0(s0); - EOF_Packet server(1, 1); - server.writePayloadImpl(out0); + EOFPacket server(1, 1); + server.writePayload(out0, sequence_id); // 2. Client reads packet ReadBufferFromString in0(s0); - PacketResponse client(server_capability_flags); - client.readPayloadImpl(in0); + ResponsePacket client(server_capability_flags); + client.readPayload(in0, sequence_id); // Check ASSERT(client.getType() == PACKET_EOF) @@ -142,12 +150,12 @@ int main(int argc, char ** argv) std::string s0; WriteBufferFromString out0(s0); ColumnDefinition server("schema", "tbl", "org_tbl", "name", "org_name", 33, 0x00, MYSQL_TYPE_STRING, 0x00, 0x00); - server.writePayloadImpl(out0); + server.writePayload(out0, sequence_id); // 2. Client reads packet ReadBufferFromString in0(s0); ColumnDefinition client; - client.readPayloadImpl(in0); + client.readPayload(in0, sequence_id); // Check ASSERT(client.column_type == server.column_type) diff --git a/src/Core/ya.make b/src/Core/ya.make index 3b45cf22f57..74a0f7e273a 100644 --- a/src/Core/ya.make +++ b/src/Core/ya.make @@ -17,15 +17,22 @@ SRCS( ExternalTable.cpp Field.cpp iostream_debug_helpers.cpp - MySQLClient.cpp - MySQLProtocol.cpp - MySQLReplication.cpp NamesAndTypes.cpp PostgreSQLProtocol.cpp Settings.cpp SettingsEnums.cpp SettingsFields.cpp SortDescription.cpp + MySQL/Authentication.cpp + MySQL/IMySQLReadPacket.cpp + MySQL/IMySQLWritePacket.cpp + MySQL/MySQLClient.cpp + MySQL/MySQLReplication.cpp + MySQL/PacketEndpoint.cpp + MySQL/PacketsConnection.cpp + MySQL/PacketsGeneric.cpp + MySQL/PacketsProtocolText.cpp + MySQL/PacketsReplication.cpp ) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 227f4b661db..9cec8179837 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -366,6 +366,8 @@ void DatabaseAtomic::loadStoredObjects(Context & context, bool has_force_restore std::lock_guard lock{mutex}; table_names = table_name_to_path; } + + Poco::File(path_to_table_symlinks).createDirectories(); for (const auto & table : table_names) tryCreateSymlink(table.first, table.second); } diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 4553e648804..1bf4db74bf0 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -18,7 +18,7 @@ #endif #if USE_MYSQL -# include +# include # include # include # include @@ -103,16 +103,16 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String const ASTFunction * engine = engine_define->engine; if (!engine->arguments || engine->arguments->children.size() != 4) throw Exception( - "MySQL Database require mysql_hostname, mysql_database_name, mysql_username, mysql_password arguments.", + engine_name + " Database require mysql_hostname, mysql_database_name, mysql_username, mysql_password arguments.", ErrorCodes::BAD_ARGUMENTS); ASTs & arguments = engine->arguments->children; arguments[1] = evaluateConstantExpressionOrIdentifierAsLiteral(arguments[1], context); - const auto & host_name_and_port = safeGetLiteralValue(arguments[0], "MySQL"); - const auto & mysql_database_name = safeGetLiteralValue(arguments[1], "MySQL"); - const auto & mysql_user_name = safeGetLiteralValue(arguments[2], "MySQL"); - const auto & mysql_user_password = safeGetLiteralValue(arguments[3], "MySQL"); + const auto & host_name_and_port = safeGetLiteralValue(arguments[0], engine_name); + const auto & mysql_database_name = safeGetLiteralValue(arguments[1], engine_name); + const auto & mysql_user_name = safeGetLiteralValue(arguments[2], engine_name); + const auto & mysql_user_password = safeGetLiteralValue(arguments[3], engine_name); try { diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp index f724faac637..ed85028d04d 100644 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ b/src/Databases/DatabaseWithDictionaries.cpp @@ -153,7 +153,6 @@ void DatabaseWithDictionaries::createDictionary(const Context & context, const S if (isTableExist(dictionary_name, global_context)) throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already exists.", dict_id.getFullTableName()); - String dictionary_metadata_path = getObjectMetadataPath(dictionary_name); String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp"; String statement = getObjectDefinitionFromCreateQuery(query); diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp index ce21d312b1f..b5231a23d7e 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp @@ -6,6 +6,7 @@ # include +# include # include # include # include diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.h b/src/Databases/MySQL/DatabaseMaterializeMySQL.h index 93548584296..799db65b481 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.h @@ -5,7 +5,7 @@ #if USE_MYSQL #include -#include +#include #include #include #include diff --git a/src/Databases/MySQL/MaterializeMetadata.h b/src/Databases/MySQL/MaterializeMetadata.h index 3fcae3dcf1c..b50e0d28334 100644 --- a/src/Databases/MySQL/MaterializeMetadata.h +++ b/src/Databases/MySQL/MaterializeMetadata.h @@ -7,7 +7,7 @@ #if USE_MYSQL #include -#include +#include #include #include diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.h b/src/Databases/MySQL/MaterializeMySQLSyncThread.h index 599efc2fb42..0327690f1b0 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.h +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.h @@ -8,7 +8,7 @@ # include # include -# include +# include # include # include # include diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index 5c7fd4f50fd..643e9af54b8 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -18,6 +18,7 @@ void registerDictionarySourceCassandra(DictionarySourceFactory & factory) [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, const Context & /* context */, + const std::string & /* default_database */, bool /*check_config*/) -> DictionarySourcePtr { #if USE_CASSANDRA diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 180750d143a..234cbddfc80 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -53,7 +53,8 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( const std::string & path_to_settings, const std::string & config_prefix, const Block & sample_block_, - const Context & context_) + const Context & context_, + const std::string & default_database) : update_time{std::chrono::system_clock::from_time_t(0)} , dict_struct{dict_struct_} , host{config.getString(config_prefix + ".host")} @@ -61,7 +62,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( , secure(config.getBool(config_prefix + ".secure", false)) , user{config.getString(config_prefix + ".user", "")} , password{config.getString(config_prefix + ".password", "")} - , db{config.getString(config_prefix + ".db", "")} + , db{config.getString(config_prefix + ".db", default_database)} , table{config.getString(config_prefix + ".table")} , where{config.getString(config_prefix + ".where", "")} , update_field{config.getString(config_prefix + ".update_field", "")} @@ -226,9 +227,11 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & context, + const std::string & default_database, bool /* check_config */) -> DictionarySourcePtr { - return std::make_unique(dict_struct, config, config_prefix, config_prefix + ".clickhouse", sample_block, context); + return std::make_unique( + dict_struct, config, config_prefix, config_prefix + ".clickhouse", sample_block, context, default_database); }; factory.registerSource("clickhouse", create_table_source); } diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 13dc0323039..276f6916ada 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -24,7 +24,8 @@ public: const std::string & path_to_settings, const std::string & config_prefix, const Block & sample_block_, - const Context & context); + const Context & context, + const std::string & default_database); /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other); diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 03b4fe106c7..c33b7b5a3ae 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -42,7 +42,8 @@ DictionaryPtr DictionaryFactory::create( const DictionaryStructure dict_struct{config, config_prefix + ".structure"}; - DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context, check_source_config); + DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create( + name, config, config_prefix + ".source", dict_struct, context, config.getString(config_prefix + ".database", ""), check_source_config); LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name); const auto & layout_type = keys.front(); diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index a9e90024745..af3552364ba 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -80,6 +80,7 @@ DictionarySourcePtr DictionarySourceFactory::create( const std::string & config_prefix, const DictionaryStructure & dict_struct, const Context & context, + const std::string & default_database, bool check_config) const { Poco::Util::AbstractConfiguration::Keys keys; @@ -96,7 +97,7 @@ DictionarySourcePtr DictionarySourceFactory::create( { const auto & create_source = found->second; auto sample_block = createSampleBlock(dict_struct); - return create_source(dict_struct, config, config_prefix, sample_block, context, check_config); + return create_source(dict_struct, config, config_prefix, sample_block, context, default_database, check_config); } throw Exception{name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG}; diff --git a/src/Dictionaries/DictionarySourceFactory.h b/src/Dictionaries/DictionarySourceFactory.h index 5d3578a7e28..1406660dfb4 100644 --- a/src/Dictionaries/DictionarySourceFactory.h +++ b/src/Dictionaries/DictionarySourceFactory.h @@ -26,12 +26,16 @@ class DictionarySourceFactory : private boost::noncopyable public: static DictionarySourceFactory & instance(); + /// 'default_database' - the database where dictionary itself was created. + /// It is used as default_database for ClickHouse dictionary source when no explicit database was specified. + /// Does not make sense for other sources. using Creator = std::function; DictionarySourceFactory(); @@ -44,6 +48,7 @@ public: const std::string & config_prefix, const DictionaryStructure & dict_struct, const Context & context, + const std::string & default_database, bool check_config) const; private: diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 38965e00a84..46df227dd67 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -220,6 +220,7 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & context, + const std::string & /* default_database */, bool check_config) -> DictionarySourcePtr { if (dict_struct.has_expressions) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index a581f77ed3a..18893a99f4e 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -76,6 +76,7 @@ void registerDictionarySourceFile(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & context, + const std::string & /* default_database */, bool check_config) -> DictionarySourcePtr { if (dict_struct.has_expressions) diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 95aab78ba2b..18a97f34486 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -197,6 +197,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & context, + const std::string & /* default_database */, bool check_config) -> DictionarySourcePtr { if (dict_struct.has_expressions) diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index ba538201910..6d763444b54 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -298,6 +298,7 @@ void registerDictionarySourceLibrary(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & context, + const std::string & /* default_database */, bool check_config) -> DictionarySourcePtr { return std::make_unique(dict_struct, config, config_prefix + ".library", sample_block, context, check_config); diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index f4376da952d..5b5d0c4d20f 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -14,6 +14,7 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) const std::string & root_config_prefix, Block & sample_block, const Context &, + const std::string & /* default_database */, bool /* check_config */) { const auto config_prefix = root_config_prefix + ".mongodb"; diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index f016f2bf537..a21b1bd50fc 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -19,6 +19,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & /* context */, + const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { #if USE_MYSQL return std::make_unique(dict_struct, config, config_prefix + ".mysql", sample_block); diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 8794f0620e2..27acd723938 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -13,6 +13,7 @@ void registerDictionarySourceRedis(DictionarySourceFactory & factory) const String & config_prefix, Block & sample_block, const Context & /* context */, + const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { return std::make_unique(dict_struct, config, config_prefix + ".redis", sample_block); }; diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index b3393d55e5d..793d8da7390 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -275,6 +275,7 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) const std::string & config_prefix, Block & sample_block, const Context & context, + const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { #if USE_ODBC BridgeHelperPtr bridge = std::make_shared>( @@ -300,6 +301,7 @@ void registerDictionarySourceJDBC(DictionarySourceFactory & factory) const std::string & /* config_prefix */, Block & /* sample_block */, const Context & /* context */, + const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { throw Exception{"Dictionary source of type `jdbc` is disabled until consistent support for nullable fields.", ErrorCodes::SUPPORT_IS_DISABLED}; diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index b3db806ca5e..b1962e48eea 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -402,7 +402,11 @@ void buildConfigurationFromFunctionWithKeyValueArguments( * * */ -void buildSourceConfiguration(AutoPtr doc, AutoPtr root, const ASTFunctionWithKeyValueArguments * source, const ASTDictionarySettings * settings) +void buildSourceConfiguration( + AutoPtr doc, + AutoPtr root, + const ASTFunctionWithKeyValueArguments * source, + const ASTDictionarySettings * settings) { AutoPtr outer_element(doc->createElement("source")); root->appendChild(outer_element); @@ -498,7 +502,9 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer bool complex = DictionaryFactory::instance().isComplex(dictionary_layout->layout_type); - auto all_attr_names_and_types = buildDictionaryAttributesConfiguration(xml_document, structure_element, query.dictionary_attributes_list, pk_attrs); + auto all_attr_names_and_types = buildDictionaryAttributesConfiguration( + xml_document, structure_element, query.dictionary_attributes_list, pk_attrs); + checkPrimaryKey(all_attr_names_and_types, pk_attrs); buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list); diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index fbfed640e22..0f696ffacd8 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -12,11 +12,15 @@ namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; + extern const int TOO_LARGE_ARRAY_SIZE; } namespace DB::GatherUtils { +inline constexpr size_t MAX_ARRAY_SIZE = 1 << 30; + + /// Methods to copy Slice to Sink, overloaded for various combinations of types. template @@ -673,6 +677,10 @@ void resizeDynamicSize(ArraySource && array_source, ValueSource && value_source, if (size >= 0) { auto length = static_cast(size); + if (length > MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size: {}, maximum: {}", + length, MAX_ARRAY_SIZE); + if (array_size <= length) { writeSlice(array_source.getWhole(), sink); @@ -685,6 +693,10 @@ void resizeDynamicSize(ArraySource && array_source, ValueSource && value_source, else { auto length = static_cast(-size); + if (length > MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size: {}, maximum: {}", + length, MAX_ARRAY_SIZE); + if (array_size <= length) { for (size_t i = array_size; i < length; ++i) @@ -714,6 +726,10 @@ void resizeConstantSize(ArraySource && array_source, ValueSource && value_source if (size >= 0) { auto length = static_cast(size); + if (length > MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size: {}, maximum: {}", + length, MAX_ARRAY_SIZE); + if (array_size <= length) { writeSlice(array_source.getWhole(), sink); @@ -726,6 +742,10 @@ void resizeConstantSize(ArraySource && array_source, ValueSource && value_source else { auto length = static_cast(-size); + if (length > MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size: {}, maximum: {}", + length, MAX_ARRAY_SIZE); + if (array_size <= length) { for (size_t i = array_size; i < length; ++i) diff --git a/src/Functions/GatherUtils/GatherUtils.h b/src/Functions/GatherUtils/GatherUtils.h index be6fae017c0..6699cc655e4 100644 --- a/src/Functions/GatherUtils/GatherUtils.h +++ b/src/Functions/GatherUtils/GatherUtils.h @@ -57,7 +57,6 @@ void sliceHas(IArraySource & first, IArraySource & second, ArraySearchType searc void push(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, bool push_front); void resizeDynamicSize(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, const IColumn & size_column); - void resizeConstantSize(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, ssize_t size); } diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 16f95ac2682..640c8701295 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -102,7 +102,8 @@ private: } template - bool executeConstStartStep(Block & block, const IColumn * end_arg, const T start, const T step, const size_t input_rows_count, const size_t result) const + bool executeConstStartStep( + Block & block, const IColumn * end_arg, const T start, const T step, const size_t input_rows_count, const size_t result) const { auto end_column = checkAndGetColumn>(end_arg); if (!end_column) @@ -145,8 +146,14 @@ private: for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx) { for (size_t st = start, ed = end_data[row_idx]; st < ed; st += step) + { out_data[offset++] = st; + if (st > st + step) + throw Exception{"A call to function " + getName() + " overflows, investigate the values of arguments you are passing", + ErrorCodes::ARGUMENT_OUT_OF_BOUND}; + } + out_offsets[row_idx] = offset; } @@ -155,7 +162,8 @@ private: } template - bool executeConstStep(Block & block, const IColumn * start_arg, const IColumn * end_arg, const T step, const size_t input_rows_count, const size_t result) const + bool executeConstStep( + Block & block, const IColumn * start_arg, const IColumn * end_arg, const T step, const size_t input_rows_count, const size_t result) const { auto start_column = checkAndGetColumn>(start_arg); auto end_column = checkAndGetColumn>(end_arg); @@ -200,8 +208,14 @@ private: for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx) { for (size_t st = start_data[row_idx], ed = end_data[row_idx]; st < ed; st += step) + { out_data[offset++] = st; + if (st > st + step) + throw Exception{"A call to function " + getName() + " overflows, investigate the values of arguments you are passing", + ErrorCodes::ARGUMENT_OUT_OF_BOUND}; + } + out_offsets[row_idx] = offset; } @@ -210,7 +224,8 @@ private: } template - bool executeConstStart(Block & block, const IColumn * end_arg, const IColumn * step_arg, const T start, const size_t input_rows_count, const size_t result) const + bool executeConstStart( + Block & block, const IColumn * end_arg, const IColumn * step_arg, const T start, const size_t input_rows_count, const size_t result) const { auto end_column = checkAndGetColumn>(end_arg); auto step_column = checkAndGetColumn>(step_arg); @@ -255,8 +270,14 @@ private: for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx) { for (size_t st = start, ed = end_data[row_idx]; st < ed; st += step_data[row_idx]) + { out_data[offset++] = st; + if (st > st + step_data[row_idx]) + throw Exception{"A call to function " + getName() + " overflows, investigate the values of arguments you are passing", + ErrorCodes::ARGUMENT_OUT_OF_BOUND}; + } + out_offsets[row_idx] = offset; } @@ -265,7 +286,9 @@ private: } template - bool executeGeneric(Block & block, const IColumn * start_col, const IColumn * end_col, const IColumn * step_col, const size_t input_rows_count, const size_t result) const + bool executeGeneric( + Block & block, const IColumn * start_col, const IColumn * end_col, const IColumn * step_col, + const size_t input_rows_count, const size_t result) const { auto start_column = checkAndGetColumn>(start_col); auto end_column = checkAndGetColumn>(end_col); @@ -313,8 +336,14 @@ private: for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx) { for (size_t st = start_data[row_idx], ed = end_start[row_idx]; st < ed; st += step_data[row_idx]) + { out_data[offset++] = st; + if (st > st + step_data[row_idx]) + throw Exception{"A call to function " + getName() + " overflows, investigate the values of arguments you are passing", + ErrorCodes::ARGUMENT_OUT_OF_BOUND}; + } + out_offsets[row_idx] = offset; } diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index 0fb9c4b6a64..9702edf7079 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -14,10 +14,13 @@ namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int PARAMETER_OUT_OF_BOUND; } + class FunctionH3KRing : public IFunction { public: @@ -65,6 +68,15 @@ public: const H3Index origin_hindex = col_hindex->getUInt(row); const int k = col_k->getInt(row); + /// Overflow is possible. The function maxKringSize does not check for overflow. + /// The calculation is similar to square of k but several times more. + /// Let's use huge underestimation as the safe bound. We should not allow to generate too large arrays nevertheless. + constexpr auto max_k = 10000; + if (k > max_k) + throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Too large 'k' argument for {} function, maximum {}", getName(), max_k); + if (k < 0) + throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Argument 'k' for {} function must be non negative", getName()); + const auto vec_size = maxKringSize(k); hindex_vec.resize(vec_size); kRing(origin_hindex, k, hindex_vec.data()); diff --git a/src/IO/MySQLPacketPayloadReadBuffer.cpp b/src/IO/MySQLPacketPayloadReadBuffer.cpp new file mode 100644 index 00000000000..16b1cd5de19 --- /dev/null +++ b/src/IO/MySQLPacketPayloadReadBuffer.cpp @@ -0,0 +1,65 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_PACKET_FROM_CLIENT; +} + +const size_t MAX_PACKET_LENGTH = (1 << 24) - 1; // 16 mb + +MySQLPacketPayloadReadBuffer::MySQLPacketPayloadReadBuffer(ReadBuffer & in_, uint8_t & sequence_id_) + : ReadBuffer(in_.position(), 0), in(in_), sequence_id(sequence_id_) // not in.buffer().begin(), because working buffer may include previous packet +{ +} + +bool MySQLPacketPayloadReadBuffer::nextImpl() +{ + if (!has_read_header || (payload_length == MAX_PACKET_LENGTH && offset == payload_length)) + { + has_read_header = true; + working_buffer.resize(0); + offset = 0; + payload_length = 0; + in.readStrict(reinterpret_cast(&payload_length), 3); + + if (payload_length > MAX_PACKET_LENGTH) + { + std::ostringstream tmp; + tmp << "Received packet with payload larger than max_packet_size: " << payload_length; + throw Exception(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + } + + size_t packet_sequence_id = 0; + in.read(reinterpret_cast(packet_sequence_id)); + if (packet_sequence_id != sequence_id) + { + std::ostringstream tmp; + tmp << "Received packet with wrong sequence-id: " << packet_sequence_id << ". Expected: " << static_cast(sequence_id) << '.'; + throw Exception(tmp.str(), ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + } + sequence_id++; + + if (payload_length == 0) + return false; + } + else if (offset == payload_length) + { + return false; + } + + in.nextIfAtEnd(); + working_buffer = ReadBuffer::Buffer(in.position(), in.buffer().end()); + size_t count = std::min(in.available(), payload_length - offset); + working_buffer.resize(count); + in.ignore(count); + + offset += count; + + return true; +} + +} diff --git a/src/IO/MySQLPacketPayloadReadBuffer.h b/src/IO/MySQLPacketPayloadReadBuffer.h new file mode 100644 index 00000000000..f90a34ba934 --- /dev/null +++ b/src/IO/MySQLPacketPayloadReadBuffer.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace DB +{ + +/** Reading packets. + * Internally, it calls (if no more data) next() method of the underlying ReadBufferFromPocoSocket, and sets the working buffer to the rest part of the current packet payload. + */ +class MySQLPacketPayloadReadBuffer : public ReadBuffer +{ +private: + ReadBuffer & in; + uint8_t & sequence_id; + + bool has_read_header = false; + + // Size of packet which is being read now. + size_t payload_length = 0; + + // Offset in packet payload. + size_t offset = 0; + +protected: + bool nextImpl() override; + +public: + MySQLPacketPayloadReadBuffer(ReadBuffer & in_, uint8_t & sequence_id_); +}; + +} + diff --git a/src/IO/MySQLPacketPayloadWriteBuffer.cpp b/src/IO/MySQLPacketPayloadWriteBuffer.cpp new file mode 100644 index 00000000000..3d803dfa070 --- /dev/null +++ b/src/IO/MySQLPacketPayloadWriteBuffer.cpp @@ -0,0 +1,61 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_WRITE_AFTER_END_OF_BUFFER; +} + +const size_t MAX_PACKET_LENGTH = (1 << 24) - 1; // 16 mb + +MySQLPacketPayloadWriteBuffer::MySQLPacketPayloadWriteBuffer(WriteBuffer & out_, size_t payload_length_, uint8_t & sequence_id_) + : WriteBuffer(out_.position(), 0), out(out_), sequence_id(sequence_id_), total_left(payload_length_) +{ + startNewPacket(); + setWorkingBuffer(); + pos = out.position(); +} + +void MySQLPacketPayloadWriteBuffer::startNewPacket() +{ + payload_length = std::min(total_left, MAX_PACKET_LENGTH); + bytes_written = 0; + total_left -= payload_length; + + out.write(reinterpret_cast(&payload_length), 3); + out.write(sequence_id++); + bytes += 4; +} + +void MySQLPacketPayloadWriteBuffer::setWorkingBuffer() +{ + out.nextIfAtEnd(); + working_buffer = WriteBuffer::Buffer(out.position(), out.position() + std::min(payload_length - bytes_written, out.available())); + + if (payload_length - bytes_written == 0) + { + /// Finished writing packet. Due to an implementation of WriteBuffer, working_buffer cannot be empty. Further write attempts will throw Exception. + eof = true; + working_buffer.resize(1); + } +} + +void MySQLPacketPayloadWriteBuffer::nextImpl() +{ + const int written = pos - working_buffer.begin(); + if (eof) + throw Exception("Cannot write after end of buffer.", ErrorCodes::CANNOT_WRITE_AFTER_END_OF_BUFFER); + + out.position() += written; + bytes_written += written; + + /// Packets of size greater than MAX_PACKET_LENGTH are split into few packets of size MAX_PACKET_LENGTH and las packet of size < MAX_PACKET_LENGTH. + if (bytes_written == payload_length && (total_left > 0 || payload_length == MAX_PACKET_LENGTH)) + startNewPacket(); + + setWorkingBuffer(); +} + +} diff --git a/src/IO/MySQLPacketPayloadWriteBuffer.h b/src/IO/MySQLPacketPayloadWriteBuffer.h new file mode 100644 index 00000000000..f54bec06dfb --- /dev/null +++ b/src/IO/MySQLPacketPayloadWriteBuffer.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace DB +{ + +/** Writing packets. + * https://dev.mysql.com/doc/internals/en/mysql-packet.html + */ +class MySQLPacketPayloadWriteBuffer : public WriteBuffer +{ +public: + MySQLPacketPayloadWriteBuffer(WriteBuffer & out_, size_t payload_length_, uint8_t & sequence_id_); + + bool remainingPayloadSize() { return total_left; } + +protected: + void nextImpl() override; + +private: + WriteBuffer & out; + uint8_t & sequence_id; + + size_t total_left = 0; + size_t payload_length = 0; + size_t bytes_written = 0; + bool eof = false; + + void startNewPacket(); + + /// Sets working buffer to the rest of current packet payload. + void setWorkingBuffer(); +}; + +} diff --git a/src/IO/ya.make b/src/IO/ya.make index d823709ce26..600112fe675 100644 --- a/src/IO/ya.make +++ b/src/IO/ya.make @@ -26,6 +26,8 @@ SRCS( MemoryReadWriteBuffer.cpp MMapReadBufferFromFile.cpp MMapReadBufferFromFileDescriptor.cpp + MySQLPacketPayloadReadBuffer.cpp + MySQLPacketPayloadWriteBuffer.cpp NullWriteBuffer.cpp parseDateTimeBestEffort.cpp PeekableReadBuffer.cpp diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 9a0ee7fed86..8bfa4801a58 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -449,7 +449,6 @@ void NO_INLINE Aggregator::executeImpl( typename Method::State state(key_columns, key_sizes, aggregation_state_cache); if (!no_more_keys) - //executeImplCase(method, state, aggregates_pool, rows, aggregate_instructions, overflow_row); executeImplBatch(method, state, aggregates_pool, rows, aggregate_instructions); else executeImplCase(method, state, aggregates_pool, rows, aggregate_instructions, overflow_row); @@ -534,22 +533,36 @@ void NO_INLINE Aggregator::executeImplBatch( /// Optimization for special case when aggregating by 8bit key. if constexpr (std::is_same_v) { + /// We use another method if there are aggregate functions with -Array combinator. + bool has_arrays = false; for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst) { - inst->batch_that->addBatchLookupTable8( - rows, - reinterpret_cast(method.data.data()), - inst->state_offset, - [&](AggregateDataPtr & aggregate_data) - { - aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); - createAggregateStates(aggregate_data); - }, - state.getKeyData(), - inst->batch_arguments, - aggregates_pool); + if (inst->offsets) + { + has_arrays = true; + break; + } + } + + if (!has_arrays) + { + for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst) + { + inst->batch_that->addBatchLookupTable8( + rows, + reinterpret_cast(method.data.data()), + inst->state_offset, + [&](AggregateDataPtr & aggregate_data) + { + aggregate_data = aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(aggregate_data); + }, + state.getKeyData(), + inst->batch_arguments, + aggregates_pool); + } + return; } - return; } /// Generic case. @@ -629,7 +642,7 @@ void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl( void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns & aggregate_columns, Columns & materialized_columns, - AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) + AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) { for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_columns[i].resize(params.aggregates[i].arguments.size()); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 3f8ff0d41f2..e0e19ae317d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -142,10 +142,6 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (create.storage->engine->name == "Atomic") { - if (!context.getSettingsRef().allow_experimental_database_atomic && !internal) - throw Exception("Atomic is an experimental database engine. " - "Enable allow_experimental_database_atomic to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); - if (create.attach && create.uuid == UUIDHelpers::Nil) throw Exception("UUID must be specified for ATTACH", ErrorCodes::INCORRECT_QUERY); else if (create.uuid == UUIDHelpers::Nil) diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 127df9b5eac..d38a3fa68dc 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes { extern const int ALIAS_REQUIRED; extern const int AMBIGUOUS_COLUMN_NAME; + extern const int LOGICAL_ERROR; } namespace @@ -187,7 +188,8 @@ StoragePtr JoinedTables::getLeftTableStorage() bool JoinedTables::resolveTables() { tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); - assert(tables_with_columns.size() == table_expressions.size()); + if (tables_with_columns.size() != table_expressions.size()) + throw Exception("Unexpected tables count", ErrorCodes::LOGICAL_ERROR); const auto & settings = context.getSettingsRef(); if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 0d3ba60640b..fb6a7239d3a 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -108,7 +108,7 @@ std::optional findFirstNonDeterministicFunctionName(const MutationComman ASTPtr prepareQueryAffectedAST(const std::vector & commands) { /// Execute `SELECT count() FROM storage WHERE predicate1 OR predicate2 OR ...` query. - /// The result can differ from tne number of affected rows (e.g. if there is an UPDATE command that + /// The result can differ from the number of affected rows (e.g. if there is an UPDATE command that /// changes how many rows satisfy the predicates of the subsequent commands). /// But we can be sure that if count = 0, then no rows will be touched. diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 9320e8eacf5..461dd997cd1 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -344,14 +344,14 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( const auto & create_materialized_column_declaration = [&](const String & name, const String & type, const auto & default_value) { - const auto column_declaration = std::make_shared(); + auto column_declaration = std::make_shared(); column_declaration->name = name; column_declaration->type = makeASTFunction(type); column_declaration->default_specifier = "MATERIALIZED"; column_declaration->default_expression = std::make_shared(default_value); column_declaration->children.emplace_back(column_declaration->type); column_declaration->children.emplace_back(column_declaration->default_expression); - return std::move(column_declaration); + return column_declaration; }; /// Add _sign and _version column. diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index 3915a0f7f43..86bdec628cd 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -15,6 +15,7 @@ namespace DB namespace ErrorCodes { + extern const int LOGICAL_ERROR; } PredicateExpressionsOptimizer::PredicateExpressionsOptimizer( @@ -111,6 +112,10 @@ bool PredicateExpressionsOptimizer::tryRewritePredicatesToTables(ASTs & tables_e { bool is_rewrite_tables = false; + if (tables_element.size() != tables_predicates.size()) + throw Exception("Unexpected elements count in predicate push down: `set enable_optimize_predicate_expression = 0` to disable", + ErrorCodes::LOGICAL_ERROR); + for (size_t index = tables_element.size(); index > 0; --index) { size_t table_pos = index - 1; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 78e9f4bba3f..0f31e698421 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -474,13 +474,16 @@ static std::tuple executeQueryImpl( bool log_queries = settings.log_queries && !internal; /// Log into system table start of query execution, if need. - if (log_queries && elem.type >= settings.log_queries_min_type) + if (log_queries) { if (settings.log_query_settings) elem.query_settings = std::make_shared(context.getSettingsRef()); - if (auto query_log = context.getQueryLog()) - query_log->add(elem); + if (elem.type >= settings.log_queries_min_type) + { + if (auto query_log = context.getQueryLog()) + query_log->add(elem); + } } /// Common code for finish and exception callbacks diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index d7c2dfa5287..bb476999f8c 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -84,9 +84,6 @@ static void loadDatabase( } -#define SYSTEM_DATABASE "system" - - void loadMetadata(Context & context, const String & default_database_name) { Poco::Logger * log = &Poco::Logger::get("loadMetadata"); @@ -114,7 +111,7 @@ void loadMetadata(Context & context, const String & default_database_name) if (endsWith(it.name(), ".sql")) { String db_name = it.name().substr(0, it.name().size() - 4); - if (db_name != SYSTEM_DATABASE) + if (db_name != DatabaseCatalog::SYSTEM_DATABASE) databases.emplace(unescapeForFileName(db_name), path + "/" + db_name); } @@ -140,7 +137,7 @@ void loadMetadata(Context & context, const String & default_database_name) if (it.name().at(0) == '.') continue; - if (it.name() == SYSTEM_DATABASE) + if (it.name() == DatabaseCatalog::SYSTEM_DATABASE) continue; databases.emplace(unescapeForFileName(it.name()), it.path().toString()); @@ -172,21 +169,20 @@ void loadMetadata(Context & context, const String & default_database_name) void loadMetadataSystem(Context & context) { - String path = context.getPath() + "metadata/" SYSTEM_DATABASE; - if (Poco::File(path).exists()) + String path = context.getPath() + "metadata/" + DatabaseCatalog::SYSTEM_DATABASE; + String metadata_file = path + ".sql"; + if (Poco::File(path).exists() || Poco::File(metadata_file).exists()) { /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. - loadDatabase(context, SYSTEM_DATABASE, path, true); + loadDatabase(context, DatabaseCatalog::SYSTEM_DATABASE, path, true); } else { /// Initialize system database manually - String global_path = context.getPath(); - Poco::File(global_path + "data/" SYSTEM_DATABASE).createDirectories(); - Poco::File(global_path + "metadata/" SYSTEM_DATABASE).createDirectories(); - - auto system_database = std::make_shared(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE "/", context); - DatabaseCatalog::instance().attachDatabase(SYSTEM_DATABASE, system_database); + String database_create_query = "CREATE DATABASE "; + database_create_query += DatabaseCatalog::SYSTEM_DATABASE; + database_create_query += " ENGINE=Atomic"; + executeCreateQuery(database_create_query, context, DatabaseCatalog::SYSTEM_DATABASE, "", true); } } diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index 7aae0a7d191..b3d4198a396 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -10,6 +9,8 @@ namespace DB { using namespace MySQLProtocol; +using namespace MySQLProtocol::Generic; +using namespace MySQLProtocol::ProtocolText; MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & settings_) @@ -29,17 +30,17 @@ void MySQLOutputFormat::initialize() if (header.columns()) { - packet_sender->sendPacket(LengthEncodedNumber(header.columns())); + packet_endpoint->sendPacket(LengthEncodedNumber(header.columns())); for (size_t i = 0; i < header.columns(); i++) { const auto & column_name = header.getColumnsWithTypeAndName()[i].name; - packet_sender->sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId())); + packet_endpoint->sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId())); } if (!(context->mysql.client_capabilities & Capability::CLIENT_DEPRECATE_EOF)) { - packet_sender->sendPacket(EOF_Packet(0, 0)); + packet_endpoint->sendPacket(EOFPacket(0, 0)); } } } @@ -52,8 +53,8 @@ void MySQLOutputFormat::consume(Chunk chunk) for (size_t i = 0; i < chunk.getNumRows(); i++) { - ProtocolText::ResultsetRow row_packet(data_types, chunk.getColumns(), i); - packet_sender->sendPacket(row_packet); + ProtocolText::ResultSetRow row_packet(data_types, chunk.getColumns(), i); + packet_endpoint->sendPacket(row_packet); } } @@ -75,17 +76,17 @@ void MySQLOutputFormat::finalize() const auto & header = getPort(PortKind::Main).getHeader(); if (header.columns() == 0) - packet_sender->sendPacket(OK_Packet(0x0, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); + packet_endpoint->sendPacket(OKPacket(0x0, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); else if (context->mysql.client_capabilities & CLIENT_DEPRECATE_EOF) - packet_sender->sendPacket(OK_Packet(0xfe, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); + packet_endpoint->sendPacket(OKPacket(0xfe, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); else - packet_sender->sendPacket(EOF_Packet(0, 0), true); + packet_endpoint->sendPacket(EOFPacket(0, 0), true); } void MySQLOutputFormat::flush() { - packet_sender->out->next(); + packet_endpoint->out->next(); } void registerOutputFormatProcessorMySQLWire(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.h b/src/Processors/Formats/Impl/MySQLOutputFormat.h index 2604e7fc42e..c0300675240 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.h +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.h @@ -3,7 +3,10 @@ #include #include -#include +#include +#include +#include +#include #include namespace DB @@ -26,8 +29,7 @@ public: void setContext(const Context & context_) { context = &context_; - packet_sender = std::make_unique(out, const_cast(context_.mysql.sequence_id)); /// TODO: fix it - packet_sender->max_packet_size = context_.mysql.max_packet_size; + packet_endpoint = std::make_unique(out, const_cast(context_.mysql.sequence_id)); /// TODO: fix it } void consume(Chunk) override; @@ -42,7 +44,7 @@ private: bool initialized = false; const Context * context = nullptr; - std::unique_ptr packet_sender; + std::unique_ptr packet_endpoint; FormatSettings format_settings; DataTypes data_types; }; diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 8596cb850d9..c394aff3f2c 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -140,16 +140,24 @@ void TabSeparatedRowInputFormat::readPrefix() if (format_settings.with_names_use_header) { String column_name; - do + for (;;) { readEscapedString(column_name, in); - addInputColumn(column_name); + if (!checkChar('\t', in)) + { + /// Check last column for \r before adding it, otherwise an error will be: + /// "Unknown field found in TSV header" + checkForCarriageReturn(in); + addInputColumn(column_name); + break; + } + else + addInputColumn(column_name); } - while (checkChar('\t', in)); + if (!in.eof()) { - checkForCarriageReturn(in); assertChar('\n', in); } } diff --git a/src/Processors/Pipe.cpp b/src/Processors/Pipe.cpp index e6749827efe..93dcd561c00 100644 --- a/src/Processors/Pipe.cpp +++ b/src/Processors/Pipe.cpp @@ -74,7 +74,7 @@ static OutputPort * uniteTotals(const OutputPortRawPtrs & ports, const Block & h if (ports.size() == 1) return ports.front(); - /// Calculate totals fro several streams. + /// Calculate totals from several streams. /// Take totals from first sources which has any, skip others. /// ->> Concat -> Limit diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 34c26fe2bad..2cf6ad546eb 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -5,16 +5,25 @@ #include #include #include -#include +#include +#include +#include +#include #include #include #include +#include +#include #include #include #include +#include +#include #include #include #include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -25,12 +34,16 @@ # include # include # include + #endif namespace DB { using namespace MySQLProtocol; +using namespace MySQLProtocol::Generic; +using namespace MySQLProtocol::ProtocolText; +using namespace MySQLProtocol::ConnectionPhase; #if USE_SSL using Poco::Net::SecureStreamSocket; @@ -45,6 +58,10 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } + +static const size_t PACKET_HEADER_SIZE = 4; +static const size_t SSL_REQUEST_PAYLOAD_SIZE = 32; + static String selectEmptyReplacementQuery(const String & query); static String showTableStatusReplacementQuery(const String & query); static String killConnectionIdReplacementQuery(const String & query); @@ -74,12 +91,13 @@ void MySQLHandler::run() in = std::make_shared(socket()); out = std::make_shared(socket()); - packet_sender = std::make_shared(*in, *out, connection_context.mysql.sequence_id); + packet_endpoint = std::make_shared(*in, *out, connection_context.mysql.sequence_id); try { - Handshake handshake(server_capability_flags, connection_id, VERSION_STRING + String("-") + VERSION_NAME, auth_plugin->getName(), auth_plugin->getAuthPluginData()); - packet_sender->sendPacket(handshake, true); + Handshake handshake(server_capability_flags, connection_id, VERSION_STRING + String("-") + VERSION_NAME, + auth_plugin->getName(), auth_plugin->getAuthPluginData(), CharacterSet::utf8_general_ci); + packet_endpoint->sendPacket(handshake, true); LOG_TRACE(log, "Sent handshake"); @@ -117,16 +135,16 @@ void MySQLHandler::run() catch (const Exception & exc) { log->log(exc); - packet_sender->sendPacket(ERR_Packet(exc.code(), "00000", exc.message()), true); + packet_endpoint->sendPacket(ERRPacket(exc.code(), "00000", exc.message()), true); } - OK_Packet ok_packet(0, handshake_response.capability_flags, 0, 0, 0); - packet_sender->sendPacket(ok_packet, true); + OKPacket ok_packet(0, handshake_response.capability_flags, 0, 0, 0); + packet_endpoint->sendPacket(ok_packet, true); while (true) { - packet_sender->resetSequenceId(); - PacketPayloadReadBuffer payload = packet_sender->getPayload(); + packet_endpoint->resetSequenceId(); + MySQLPacketPayloadReadBuffer payload = packet_endpoint->getPayload(); char command = 0; payload.readStrict(command); @@ -166,7 +184,7 @@ void MySQLHandler::run() } catch (...) { - packet_sender->sendPacket(ERR_Packet(getCurrentExceptionCode(), "00000", getCurrentExceptionMessage(false)), true); + packet_endpoint->sendPacket(ERRPacket(getCurrentExceptionCode(), "00000", getCurrentExceptionMessage(false)), true); } } } @@ -180,7 +198,7 @@ void MySQLHandler::run() * Reading is performed from socket instead of ReadBuffer to prevent reading part of SSL handshake. * If we read it from socket, it will be impossible to start SSL connection using Poco. Size of SSLRequest packet payload is 32 bytes, thus we can read at most 36 bytes. */ -void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) +void MySQLHandler::finishHandshake(MySQLProtocol::ConnectionPhase::HandshakeResponse & packet) { size_t packet_size = PACKET_HEADER_SIZE + SSL_REQUEST_PAYLOAD_SIZE; @@ -215,11 +233,11 @@ void MySQLHandler::finishHandshake(MySQLProtocol::HandshakeResponse & packet) packet_size = PACKET_HEADER_SIZE + payload_size; WriteBufferFromOwnString buf_for_handshake_response; buf_for_handshake_response.write(buf, pos); - copyData(*packet_sender->in, buf_for_handshake_response, packet_size - pos); + copyData(*packet_endpoint->in, buf_for_handshake_response, packet_size - pos); ReadBufferFromString payload(buf_for_handshake_response.str()); payload.ignore(PACKET_HEADER_SIZE); - packet.readPayloadImpl(payload); - packet_sender->sequence_id++; + packet.readPayloadWithUnpacked(payload); + packet_endpoint->sequence_id++; } } @@ -236,12 +254,12 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl } std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; - auth_plugin->authenticate(user_name, auth_response, connection_context, packet_sender, secure_connection, socket().peerAddress()); + auth_plugin->authenticate(user_name, auth_response, connection_context, packet_endpoint, secure_connection, socket().peerAddress()); } catch (const Exception & exc) { LOG_ERROR(log, "Authentication for user {} failed.", user_name); - packet_sender->sendPacket(ERR_Packet(exc.code(), "00000", exc.message()), true); + packet_endpoint->sendPacket(ERRPacket(exc.code(), "00000", exc.message()), true); throw; } LOG_INFO(log, "Authentication for user {} succeeded.", user_name); @@ -253,13 +271,13 @@ void MySQLHandler::comInitDB(ReadBuffer & payload) readStringUntilEOF(database, payload); LOG_DEBUG(log, "Setting current database to {}", database); connection_context.setCurrentDatabase(database); - packet_sender->sendPacket(OK_Packet(0, client_capability_flags, 0, 0, 1), true); + packet_endpoint->sendPacket(OKPacket(0, client_capability_flags, 0, 0, 1), true); } void MySQLHandler::comFieldList(ReadBuffer & payload) { ComFieldList packet; - packet.readPayloadImpl(payload); + packet.readPayloadWithUnpacked(payload); String database = connection_context.getCurrentDatabase(); StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table}, connection_context); auto metadata_snapshot = table_ptr->getInMemoryMetadataPtr(); @@ -268,14 +286,14 @@ void MySQLHandler::comFieldList(ReadBuffer & payload) ColumnDefinition column_definition( database, packet.table, packet.table, column.name, column.name, CharacterSet::binary, 100, ColumnType::MYSQL_TYPE_STRING, 0, 0 ); - packet_sender->sendPacket(column_definition); + packet_endpoint->sendPacket(column_definition); } - packet_sender->sendPacket(OK_Packet(0xfe, client_capability_flags, 0, 0, 0), true); + packet_endpoint->sendPacket(OKPacket(0xfe, client_capability_flags, 0, 0, 0), true); } void MySQLHandler::comPing() { - packet_sender->sendPacket(OK_Packet(0x0, client_capability_flags, 0, 0, 0), true); + packet_endpoint->sendPacket(OKPacket(0x0, client_capability_flags, 0, 0, 0), true); } static bool isFederatedServerSetupSetCommand(const String & query); @@ -288,7 +306,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload) // As Clickhouse doesn't support these statements, we just send OK packet in response. if (isFederatedServerSetupSetCommand(query)) { - packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); + packet_endpoint->sendPacket(OKPacket(0x00, client_capability_flags, 0, 0, 0), true); } else { @@ -318,7 +336,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload) ); if (!with_output) - packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); + packet_endpoint->sendPacket(OKPacket(0x00, client_capability_flags, 0, 0, 0), true); } } @@ -327,7 +345,9 @@ void MySQLHandler::authPluginSSL() throw Exception("ClickHouse was built without SSL support. Try specifying password using double SHA1 in users.xml.", ErrorCodes::SUPPORT_IS_DISABLED); } -void MySQLHandler::finishHandshakeSSL([[maybe_unused]] size_t packet_size, [[maybe_unused]] char * buf, [[maybe_unused]] size_t pos, [[maybe_unused]] std::function read_bytes, [[maybe_unused]] MySQLProtocol::HandshakeResponse & packet) +void MySQLHandler::finishHandshakeSSL( + [[maybe_unused]] size_t packet_size, [[maybe_unused]] char * buf, [[maybe_unused]] size_t pos, + [[maybe_unused]] std::function read_bytes, [[maybe_unused]] MySQLProtocol::ConnectionPhase::HandshakeResponse & packet) { throw Exception("Client requested SSL, while it is disabled.", ErrorCodes::SUPPORT_IS_DISABLED); } @@ -344,13 +364,15 @@ void MySQLHandlerSSL::authPluginSSL() auth_plugin = std::make_unique(public_key, private_key, log); } -void MySQLHandlerSSL::finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet) +void MySQLHandlerSSL::finishHandshakeSSL( + size_t packet_size, char *buf, size_t pos, std::function read_bytes, + MySQLProtocol::ConnectionPhase::HandshakeResponse & packet) { read_bytes(packet_size); /// Reading rest SSLRequest. SSLRequest ssl_request; ReadBufferFromMemory payload(buf, pos); payload.ignore(PACKET_HEADER_SIZE); - ssl_request.readPayloadImpl(payload); + ssl_request.readPayloadWithUnpacked(payload); connection_context.mysql.client_capabilities = ssl_request.capability_flags; connection_context.mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; secure_connection = true; @@ -358,9 +380,8 @@ void MySQLHandlerSSL::finishHandshakeSSL(size_t packet_size, char * buf, size_t in = std::make_shared(*ss); out = std::make_shared(*ss); connection_context.mysql.sequence_id = 2; - packet_sender = std::make_shared(*in, *out, connection_context.mysql.sequence_id); - packet_sender->max_packet_size = connection_context.mysql.max_packet_size; - packet_sender->receivePacket(packet); /// Reading HandshakeResponse from secure socket. + packet_endpoint = std::make_shared(*in, *out, connection_context.mysql.sequence_id); + packet_endpoint->receivePacket(packet); /// Reading HandshakeResponse from secure socket. } #endif diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index f7596850a8b..5568805cdfe 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -3,7 +3,10 @@ #include #include #include -#include +#include +#include +#include +#include #include "IServer.h" #if !defined(ARCADIA_BUILD) @@ -33,7 +36,7 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::MySQLConnection}; /// Enables SSL, if client requested. - void finishHandshake(MySQLProtocol::HandshakeResponse &); + void finishHandshake(MySQLProtocol::ConnectionPhase::HandshakeResponse &); void comQuery(ReadBuffer & payload); @@ -46,7 +49,7 @@ private: void authenticate(const String & user_name, const String & auth_plugin_name, const String & auth_response); virtual void authPluginSSL(); - virtual void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet); + virtual void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::ConnectionPhase::HandshakeResponse & packet); IServer & server; @@ -55,7 +58,7 @@ protected: Context connection_context; - std::shared_ptr packet_sender; + std::shared_ptr packet_endpoint; private: size_t connection_id = 0; @@ -85,7 +88,10 @@ public: private: void authPluginSSL() override; - void finishHandshakeSSL(size_t packet_size, char * buf, size_t pos, std::function read_bytes, MySQLProtocol::HandshakeResponse & packet) override; + + void finishHandshakeSSL( + size_t packet_size, char * buf, size_t pos, + std::function read_bytes, MySQLProtocol::ConnectionPhase::HandshakeResponse & packet) override; RSA & public_key; RSA & private_key; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 1f05476a684..bbefba70c58 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1225,7 +1225,8 @@ void MergeTreeData::rename(const String & new_table_path, const StorageID & new_ disk->moveDirectory(relative_data_path, new_table_path); } - global_context.dropCaches(); + if (!getStorageID().hasUUID()) + global_context.dropCaches(); relative_data_path = new_table_path; renameInMemory(new_table_id); @@ -1244,7 +1245,10 @@ void MergeTreeData::dropAllData() data_parts_indexes.clear(); column_sizes.clear(); - global_context.dropCaches(); + /// Tables in atomic databases have UUID and stored in persistent locations. + /// No need to drop caches (that are keyed by filesystem path) because collision is not possible. + if (!getStorageID().hasUUID()) + global_context.dropCaches(); LOG_TRACE(log, "dropAllData: removing data from filesystem."); @@ -1252,7 +1256,17 @@ void MergeTreeData::dropAllData() clearPartsFromFilesystem(all_parts); for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) - disk->removeRecursive(path); + { + try + { + disk->removeRecursive(path); + } + catch (const Poco::FileNotFoundException &) + { + /// If the file is already deleted, log the error message and do nothing. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } LOG_TRACE(log, "dropAllData: done."); } diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 3d0c0327da5..9aaf894a0cb 100644 --- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -291,7 +291,8 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx Block result = granule->block; actions->execute(result); - auto column = result.getByName(expression_ast->getColumnName()).column->convertToFullColumnIfLowCardinality(); + auto column + = result.getByName(expression_ast->getColumnName()).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality(); const auto * col_uint8 = typeid_cast(column.get()); const NullMap * null_map = nullptr; diff --git a/src/Storages/StorageMaterializeMySQL.cpp b/src/Storages/StorageMaterializeMySQL.cpp index d6a99e73667..4eeb4975185 100644 --- a/src/Storages/StorageMaterializeMySQL.cpp +++ b/src/Storages/StorageMaterializeMySQL.cpp @@ -85,7 +85,10 @@ Pipe StorageMaterializeMySQL::read( auto syntax = TreeRewriter(context).analyze(expressions, pipe_header.getNamesAndTypesList()); ExpressionActionsPtr expression_actions = ExpressionAnalyzer(expressions, syntax, context).getActions(true); - pipe.addTransform(std::make_shared(pipe.getHeader(), expression_actions, filter_column_name, false)); + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, expression_actions, filter_column_name, false); + }); } return pipe; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index a98d789a048..2d96a59392b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -37,6 +37,27 @@ namespace ErrorCodes extern const int SAMPLING_NOT_SUPPORTED; } +namespace +{ + +/// Rewrite original query removing joined tables from it +void removeJoin(const ASTSelectQuery & select) +{ + const auto & tables = select.tables(); + if (!tables || tables->children.size() < 2) + return; + + const auto & joined_table = tables->children[1]->as(); + if (!joined_table.table_join) + return; + + /// The most simple temporary solution: leave only the first table in query. + /// TODO: we also need to remove joined columns and related functions (taking in account aliases if any). + tables->children.resize(1); +} + +} + StorageMerge::StorageMerge( const StorageID & table_id_, @@ -243,6 +264,9 @@ Pipe StorageMerge::createSources( SelectQueryInfo modified_query_info = query_info; modified_query_info.query = query_info.query->clone(); + /// Original query could contain JOIN but we need only the first joined table and its columns. + removeJoin(*modified_query_info.query->as()); + VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", table_name); Pipe pipe; @@ -428,9 +452,14 @@ Block StorageMerge::getQueryHeader( } case QueryProcessingStage::WithMergeableState: case QueryProcessingStage::Complete: - return InterpreterSelectQuery( - query_info.query, context, std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID())), - SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + { + auto query = query_info.query->clone(); + removeJoin(*query->as()); + + auto stream = std::make_shared( + metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID())); + return InterpreterSelectQuery(query, context, stream, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + } } throw Exception("Logical Error: unknown processed stage.", ErrorCodes::LOGICAL_ERROR); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index afaf6099937..8b233fb2408 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -40,6 +40,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int INCORRECT_DATA; extern const int CANNOT_ASSIGN_OPTIMIZE; + extern const int TIMEOUT_EXCEEDED; } namespace ActionLocks @@ -138,7 +139,6 @@ void StorageMergeTree::shutdown() mutation_wait_event.notify_all(); } - merger_mutator.merges_blocker.cancelForever(); parts_mover.moves_blocker.cancelForever(); @@ -148,7 +148,6 @@ void StorageMergeTree::shutdown() if (moving_task_handle) global_context.getBackgroundMovePool().removeTask(moving_task_handle); - try { /// We clear all old parts after stopping all background operations. @@ -220,7 +219,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, cons { /// Asks to complete merges and does not allow them to start. /// This protects against "revival" of data for a removed partition after completion of merge. - auto merge_blocker = merger_mutator.merges_blocker.cancel(); + auto merge_blocker = stopMergesAndWait(); /// NOTE: It's assumed that this method is called under lockForAlter. @@ -1140,12 +1139,38 @@ Pipe StorageMergeTree::alterPartition( return {}; } + +ActionLock StorageMergeTree::stopMergesAndWait() +{ + /// Asks to complete merges and does not allow them to start. + /// This protects against "revival" of data for a removed partition after completion of merge. + auto merge_blocker = merger_mutator.merges_blocker.cancel(); + + { + std::unique_lock lock(currently_processing_in_background_mutex); + while (!currently_merging_mutating_parts.empty()) + { + LOG_DEBUG(log, "Waiting for currently running merges ({} parts are merging right now)", + currently_merging_mutating_parts.size()); + + if (std::cv_status::timeout == currently_processing_in_background_condition.wait_for( + lock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC))) + { + throw Exception("Timeout while waiting for already running merges", ErrorCodes::TIMEOUT_EXCEEDED); + } + } + } + + return merge_blocker; +} + + void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, const Context & context) { { /// Asks to complete merges and does not allow them to start. /// This protects against "revival" of data for a removed partition after completion of merge. - auto merge_blocker = merger_mutator.merges_blocker.cancel(); + auto merge_blocker = stopMergesAndWait(); auto metadata_snapshot = getInMemoryMetadataPtr(); String partition_id = getPartitionIDFromQuery(partition, context); @@ -1175,7 +1200,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, cons } -PartitionCommandsResultInfo StorageMergeTree::attachPartition( +PartitionCommandsResultInfo StorageMergeTree::attachPartition( const ASTPtr & partition, bool attach_part, const Context & context) { PartitionCommandsResultInfo results; diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 66d6c64d705..5662f9e0088 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -128,6 +128,8 @@ private: */ bool merge(bool aggressive, const String & partition_id, bool final, bool deduplicate, String * out_disable_reason = nullptr); + ActionLock stopMergesAndWait(); + BackgroundProcessingPoolTaskResult movePartsTask(); /// Allocate block number for new mutation, write mutation to disk diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1c0c0b1f351..cac2219b062 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -268,16 +268,28 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( { bool is_first_replica = createTableIfNotExists(metadata_snapshot); - /// We have to check granularity on other replicas. If it's fixed we - /// must create our new replica with fixed granularity and store this - /// information in /replica/metadata. - other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper(); + try + { + /// NOTE If it's the first replica, these requests to ZooKeeper look redundant, we already know everything. - checkTableStructure(zookeeper_path, metadata_snapshot); + /// We have to check granularity on other replicas. If it's fixed we + /// must create our new replica with fixed granularity and store this + /// information in /replica/metadata. + other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper(); - Coordination::Stat metadata_stat; - current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); - metadata_version = metadata_stat.version; + checkTableStructure(zookeeper_path, metadata_snapshot); + + Coordination::Stat metadata_stat; + current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); + metadata_version = metadata_stat.version; + } + catch (Coordination::Exception & e) + { + if (!is_first_replica && e.code == Coordination::Error::ZNONODE) + throw Exception("Table " + zookeeper_path + " was suddenly removed.", ErrorCodes::ALL_REPLICAS_LOST); + else + throw; + } if (!is_first_replica) createReplica(metadata_snapshot); @@ -291,7 +303,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } else { - /// In old tables this node may missing or be empty String replica_metadata; bool replica_metadata_exists = current_zookeeper->tryGet(replica_path + "/metadata", replica_metadata); @@ -395,7 +406,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( } /// It maybe already removed from zk, but local in-memory mutations - /// state was not update. + /// state was not updated. if (!getZooKeeper()->exists(zookeeper_path + "/mutations/" + mutation_id)) { throw Exception(ErrorCodes::UNFINISHED, "Mutation {} was killed, manually removed or table was dropped", mutation_id); @@ -758,9 +769,9 @@ void StorageReplicatedMergeTree::dropReplica(zkutil::ZooKeeperPtr zookeeper, con } -/** Verify that list of columns and table storage_settings_ptr match those specified in ZK (/ metadata). - * If not, throw an exception. - */ +/** Verify that list of columns and table storage_settings_ptr match those specified in ZK (/metadata). + * If not, throw an exception. + */ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot) { auto zookeeper = getZooKeeper(); diff --git a/src/Storages/System/IStorageSystemOneBlock.h b/src/Storages/System/IStorageSystemOneBlock.h index b30d6476b22..81650d669dc 100644 --- a/src/Storages/System/IStorageSystemOneBlock.h +++ b/src/Storages/System/IStorageSystemOneBlock.h @@ -21,7 +21,11 @@ protected: virtual void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const = 0; public: - IStorageSystemOneBlock(const String & name_) : IStorage({"system", name_}) +#if defined(ARCADIA_BUILD) + IStorageSystemOneBlock(const String & name_) : IStorageSystemOneBlock(StorageID{"system", name_}) {} +#endif + + IStorageSystemOneBlock(const StorageID & table_id_) : IStorage(table_id_) { StorageInMemoryMetadata metadata_; metadata_.setColumns(ColumnsDescription(Self::getNamesAndTypes())); diff --git a/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/src/Storages/System/StorageSystemAsynchronousMetrics.cpp index 059ef708a81..8dabac4fb49 100644 --- a/src/Storages/System/StorageSystemAsynchronousMetrics.cpp +++ b/src/Storages/System/StorageSystemAsynchronousMetrics.cpp @@ -16,8 +16,8 @@ NamesAndTypesList StorageSystemAsynchronousMetrics::getNamesAndTypes() } -StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::string & name_, const AsynchronousMetrics & async_metrics_) - : IStorageSystemOneBlock(name_), async_metrics(async_metrics_) +StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const StorageID & table_id_, const AsynchronousMetrics & async_metrics_) + : IStorageSystemOneBlock(table_id_), async_metrics(async_metrics_) { } diff --git a/src/Storages/System/StorageSystemAsynchronousMetrics.h b/src/Storages/System/StorageSystemAsynchronousMetrics.h index f31450fe086..a2a92d248d8 100644 --- a/src/Storages/System/StorageSystemAsynchronousMetrics.h +++ b/src/Storages/System/StorageSystemAsynchronousMetrics.h @@ -25,7 +25,13 @@ private: const AsynchronousMetrics & async_metrics; protected: - StorageSystemAsynchronousMetrics(const std::string & name_, const AsynchronousMetrics & async_metrics_); +#if defined(ARCADIA_BUILD) + StorageSystemAsynchronousMetrics(const String & name_, const AsynchronousMetrics & async_metrics_) + : StorageSystemAsynchronousMetrics(StorageID{"system", name_}, async_metrics_) + { + } +#endif + StorageSystemAsynchronousMetrics(const StorageID & table_id_, const AsynchronousMetrics & async_metrics_); void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; }; diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 18cc46c0c1b..d737eff3062 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -23,8 +23,8 @@ namespace ErrorCodes extern const int TABLE_IS_DROPPED; } -StorageSystemColumns::StorageSystemColumns(const std::string & name_) - : IStorage({"system", name_}) +StorageSystemColumns::StorageSystemColumns(const StorageID & table_id_) + : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(ColumnsDescription( diff --git a/src/Storages/System/StorageSystemColumns.h b/src/Storages/System/StorageSystemColumns.h index 480afe5c36c..d90cec763c9 100644 --- a/src/Storages/System/StorageSystemColumns.h +++ b/src/Storages/System/StorageSystemColumns.h @@ -27,7 +27,7 @@ public: unsigned num_streams) override; protected: - StorageSystemColumns(const std::string & name_); + StorageSystemColumns(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 5ad9a7bc5b1..5a233ed7a33 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -12,83 +12,61 @@ namespace DB { -/** - * Implements system table 'detached_parts' which allows to get information - * about detached data parts for tables of MergeTree family. - * We don't use StorageSystemPartsBase, because it introduces virtual _state - * column and column aliases which we don't need. - */ -class StorageSystemDetachedParts final : - public ext::shared_ptr_helper, - public IStorage +StorageSystemDetachedParts::StorageSystemDetachedParts(const StorageID & table_id_) + : IStorage(table_id_) { - friend struct ext::shared_ptr_helper; -public: - std::string getName() const override { return "SystemDetachedParts"; } + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(ColumnsDescription{{ + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"partition_id", std::make_shared(std::make_shared())}, + {"name", std::make_shared()}, + {"disk", std::make_shared()}, + {"reason", std::make_shared(std::make_shared())}, + {"min_block_number", std::make_shared(std::make_shared())}, + {"max_block_number", std::make_shared(std::make_shared())}, + {"level", std::make_shared(std::make_shared())} + }}); + setInMemoryMetadata(storage_metadata); +} -protected: - explicit StorageSystemDetachedParts() - : IStorage({"system", "detached_parts"}) +Pipe StorageSystemDetachedParts::read( + const Names & /* column_names */, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum /*processed_stage*/, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) +{ + StoragesInfoStream stream(query_info, context); + + /// Create the result. + Block block = metadata_snapshot->getSampleBlock(); + MutableColumns new_columns = block.cloneEmptyColumns(); + + while (StoragesInfo info = stream.next()) { - StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(ColumnsDescription{{ - {"database", std::make_shared()}, - {"table", std::make_shared()}, - {"partition_id", std::make_shared(std::make_shared())}, - {"name", std::make_shared()}, - {"disk", std::make_shared()}, - {"reason", std::make_shared(std::make_shared())}, - {"min_block_number", std::make_shared(std::make_shared())}, - {"max_block_number", std::make_shared(std::make_shared())}, - {"level", std::make_shared(std::make_shared())} - }}); - setInMemoryMetadata(storage_metadata); - } - - Pipe read( - const Names & /* column_names */, - const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum /*processed_stage*/, - const size_t /*max_block_size*/, - const unsigned /*num_streams*/) override - { - StoragesInfoStream stream(query_info, context); - - /// Create the result. - Block block = metadata_snapshot->getSampleBlock(); - MutableColumns new_columns = block.cloneEmptyColumns(); - - while (StoragesInfo info = stream.next()) + const auto parts = info.data->getDetachedParts(); + for (const auto & p : parts) { - const auto parts = info.data->getDetachedParts(); - for (const auto & p : parts) - { - size_t i = 0; - new_columns[i++]->insert(info.database); - new_columns[i++]->insert(info.table); - new_columns[i++]->insert(p.valid_name ? p.partition_id : Field()); - new_columns[i++]->insert(p.dir_name); - new_columns[i++]->insert(p.disk); - new_columns[i++]->insert(p.valid_name ? p.prefix : Field()); - new_columns[i++]->insert(p.valid_name ? p.min_block : Field()); - new_columns[i++]->insert(p.valid_name ? p.max_block : Field()); - new_columns[i++]->insert(p.valid_name ? p.level : Field()); - } + size_t i = 0; + new_columns[i++]->insert(info.database); + new_columns[i++]->insert(info.table); + new_columns[i++]->insert(p.valid_name ? p.partition_id : Field()); + new_columns[i++]->insert(p.dir_name); + new_columns[i++]->insert(p.disk); + new_columns[i++]->insert(p.valid_name ? p.prefix : Field()); + new_columns[i++]->insert(p.valid_name ? p.min_block : Field()); + new_columns[i++]->insert(p.valid_name ? p.max_block : Field()); + new_columns[i++]->insert(p.valid_name ? p.level : Field()); } - - UInt64 num_rows = new_columns.at(0)->size(); - Chunk chunk(std::move(new_columns), num_rows); - - return Pipe(std::make_shared(std::move(block), std::move(chunk))); } -}; -StoragePtr -createDetachedPartsTable() -{ - return StorageSystemDetachedParts::create(); + UInt64 num_rows = new_columns.at(0)->size(); + Chunk chunk(std::move(new_columns), num_rows); + + return Pipe(std::make_shared(std::move(block), std::move(chunk))); } } diff --git a/src/Storages/System/StorageSystemDetachedParts.h b/src/Storages/System/StorageSystemDetachedParts.h index 3fea29e5b42..c0f1db51642 100644 --- a/src/Storages/System/StorageSystemDetachedParts.h +++ b/src/Storages/System/StorageSystemDetachedParts.h @@ -1,10 +1,35 @@ #pragma once -#include +#include namespace DB { -StoragePtr createDetachedPartsTable(); +/** + * Implements system table 'detached_parts' which allows to get information + * about detached data parts for tables of MergeTree family. + * We don't use StorageSystemPartsBase, because it introduces virtual _state + * column and column aliases which we don't need. + */ +class StorageSystemDetachedParts final : + public ext::shared_ptr_helper, + public IStorage +{ + friend struct ext::shared_ptr_helper; +public: + std::string getName() const override { return "SystemDetachedParts"; } + +protected: + explicit StorageSystemDetachedParts(const StorageID & table_id_); + + Pipe read( + const Names & /* column_names */, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum /*processed_stage*/, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) override; +}; } diff --git a/src/Storages/System/StorageSystemDisks.cpp b/src/Storages/System/StorageSystemDisks.cpp index b54f3510eb6..ed831927f16 100644 --- a/src/Storages/System/StorageSystemDisks.cpp +++ b/src/Storages/System/StorageSystemDisks.cpp @@ -11,8 +11,8 @@ namespace ErrorCodes } -StorageSystemDisks::StorageSystemDisks(const std::string & name_) - : IStorage({"system", name_}) +StorageSystemDisks::StorageSystemDisks(const StorageID & table_id_) + : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(ColumnsDescription( diff --git a/src/Storages/System/StorageSystemDisks.h b/src/Storages/System/StorageSystemDisks.h index e2fb0e03705..d2075c3c784 100644 --- a/src/Storages/System/StorageSystemDisks.h +++ b/src/Storages/System/StorageSystemDisks.h @@ -30,7 +30,7 @@ public: unsigned num_streams) override; protected: - StorageSystemDisks(const std::string & name_); + StorageSystemDisks(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemOne.cpp b/src/Storages/System/StorageSystemOne.cpp index 158388f2730..57b87e165a9 100644 --- a/src/Storages/System/StorageSystemOne.cpp +++ b/src/Storages/System/StorageSystemOne.cpp @@ -11,8 +11,8 @@ namespace DB { -StorageSystemOne::StorageSystemOne(const std::string & name_) - : IStorage({"system", name_}) +StorageSystemOne::StorageSystemOne(const StorageID & table_id_) + : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(ColumnsDescription({{"dummy", std::make_shared()}})); diff --git a/src/Storages/System/StorageSystemOne.h b/src/Storages/System/StorageSystemOne.h index 09db9822914..3469d6ccb29 100644 --- a/src/Storages/System/StorageSystemOne.h +++ b/src/Storages/System/StorageSystemOne.h @@ -31,7 +31,7 @@ public: unsigned num_streams) override; protected: - StorageSystemOne(const std::string & name_); + StorageSystemOne(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 7c0da07ed88..a3fb2cf10e6 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -15,8 +15,8 @@ namespace DB { -StorageSystemParts::StorageSystemParts(const std::string & name_) - : StorageSystemPartsBase(name_, +StorageSystemParts::StorageSystemParts(const StorageID & table_id_) + : StorageSystemPartsBase(table_id_, { {"partition", std::make_shared()}, {"name", std::make_shared()}, diff --git a/src/Storages/System/StorageSystemParts.h b/src/Storages/System/StorageSystemParts.h index 06f4f414c88..2de4cd3f3a4 100644 --- a/src/Storages/System/StorageSystemParts.h +++ b/src/Storages/System/StorageSystemParts.h @@ -19,7 +19,7 @@ public: std::string getName() const override { return "SystemParts"; } protected: - explicit StorageSystemParts(const std::string & name_); + explicit StorageSystemParts(const StorageID & table_id_); void processNextStorage(MutableColumns & columns, const StoragesInfo & info, bool has_state_column) override; }; diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index b6e94993468..f590244116d 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -258,8 +258,8 @@ Pipe StorageSystemPartsBase::read( } -StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_) - : IStorage(StorageID{"system", name_}) +StorageSystemPartsBase::StorageSystemPartsBase(const StorageID & table_id_, NamesAndTypesList && columns_) + : IStorage(table_id_) { ColumnsDescription tmp_columns(std::move(columns_)); diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index 040a03770ac..2cb19f8f17d 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -72,7 +72,7 @@ private: protected: const FormatSettings format_settings; - StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_); + StorageSystemPartsBase(const StorageID & table_id_, NamesAndTypesList && columns_); virtual void processNextStorage(MutableColumns & columns, const StoragesInfo & info, bool has_state_column) = 0; }; diff --git a/src/Storages/System/StorageSystemPartsColumns.cpp b/src/Storages/System/StorageSystemPartsColumns.cpp index 4631bb9c4c5..0851e389e4f 100644 --- a/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/src/Storages/System/StorageSystemPartsColumns.cpp @@ -15,8 +15,8 @@ namespace DB { -StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name_) - : StorageSystemPartsBase(name_, +StorageSystemPartsColumns::StorageSystemPartsColumns(const StorageID & table_id_) + : StorageSystemPartsBase(table_id_, { {"partition", std::make_shared()}, {"name", std::make_shared()}, diff --git a/src/Storages/System/StorageSystemPartsColumns.h b/src/Storages/System/StorageSystemPartsColumns.h index 596fd7dae45..6347a418875 100644 --- a/src/Storages/System/StorageSystemPartsColumns.h +++ b/src/Storages/System/StorageSystemPartsColumns.h @@ -21,7 +21,7 @@ public: std::string getName() const override { return "SystemPartsColumns"; } protected: - StorageSystemPartsColumns(const std::string & name_); + StorageSystemPartsColumns(const StorageID & table_id_); void processNextStorage(MutableColumns & columns, const StoragesInfo & info, bool has_state_column) override; }; diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index f049b69a7ff..f28ecdb7f65 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -16,8 +16,8 @@ namespace DB { -StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) - : IStorage({"system", name_}) +StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_) + : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(ColumnsDescription({ diff --git a/src/Storages/System/StorageSystemReplicas.h b/src/Storages/System/StorageSystemReplicas.h index bc830721a23..c198cc29ddc 100644 --- a/src/Storages/System/StorageSystemReplicas.h +++ b/src/Storages/System/StorageSystemReplicas.h @@ -28,7 +28,7 @@ public: unsigned num_streams) override; protected: - StorageSystemReplicas(const std::string & name_); + StorageSystemReplicas(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index bd9915237d4..9edcb1ede47 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -126,8 +126,8 @@ namespace } -StorageSystemStackTrace::StorageSystemStackTrace(const String & name_) - : IStorageSystemOneBlock(name_) +StorageSystemStackTrace::StorageSystemStackTrace(const StorageID & table_id_) + : IStorageSystemOneBlock(table_id_) { notification_pipe.open(); diff --git a/src/Storages/System/StorageSystemStackTrace.h b/src/Storages/System/StorageSystemStackTrace.h index 3f20b1e973c..a389f02eb09 100644 --- a/src/Storages/System/StorageSystemStackTrace.h +++ b/src/Storages/System/StorageSystemStackTrace.h @@ -23,7 +23,7 @@ public: String getName() const override { return "SystemStackTrace"; } static NamesAndTypesList getNamesAndTypes(); - StorageSystemStackTrace(const String & name_); + StorageSystemStackTrace(const StorageID & table_id_); protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/src/Storages/System/StorageSystemStoragePolicies.cpp b/src/Storages/System/StorageSystemStoragePolicies.cpp index 8d0a51c2fbb..c8d266644eb 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -17,8 +17,8 @@ namespace ErrorCodes } -StorageSystemStoragePolicies::StorageSystemStoragePolicies(const std::string & name_) - : IStorage({"system", name_}) +StorageSystemStoragePolicies::StorageSystemStoragePolicies(const StorageID & table_id_) + : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns( diff --git a/src/Storages/System/StorageSystemStoragePolicies.h b/src/Storages/System/StorageSystemStoragePolicies.h index 73fda1e8c46..15e5e497785 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.h +++ b/src/Storages/System/StorageSystemStoragePolicies.h @@ -30,7 +30,7 @@ public: unsigned num_streams) override; protected: - StorageSystemStoragePolicies(const std::string & name_); + StorageSystemStoragePolicies(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 1ced3a89998..5b7dad836e9 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -30,8 +30,8 @@ namespace ErrorCodes } -StorageSystemTables::StorageSystemTables(const std::string & name_) - : IStorage({"system", name_}) +StorageSystemTables::StorageSystemTables(const StorageID & table_id_) + : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(ColumnsDescription( diff --git a/src/Storages/System/StorageSystemTables.h b/src/Storages/System/StorageSystemTables.h index 766c3046dbc..259eb096ea7 100644 --- a/src/Storages/System/StorageSystemTables.h +++ b/src/Storages/System/StorageSystemTables.h @@ -28,7 +28,7 @@ public: unsigned num_streams) override; protected: - StorageSystemTables(const std::string & name_); + StorageSystemTables(const StorageID & table_id_); }; } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 2b52f0fe5cc..758feca7713 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -66,79 +67,80 @@ namespace DB { -///TODO allow store system tables in DatabaseAtomic void attachSystemTablesLocal(IDatabase & system_database) { - system_database.attachTable("one", StorageSystemOne::create("one")); - system_database.attachTable("numbers", StorageSystemNumbers::create(StorageID("system", "numbers"), false)); - system_database.attachTable("numbers_mt", StorageSystemNumbers::create(StorageID("system", "numbers_mt"), true)); - system_database.attachTable("zeros", StorageSystemZeros::create(StorageID("system", "zeros"), false)); - system_database.attachTable("zeros_mt", StorageSystemZeros::create(StorageID("system", "zeros_mt"), true)); - system_database.attachTable("databases", StorageSystemDatabases::create("databases")); - system_database.attachTable("tables", StorageSystemTables::create("tables")); - system_database.attachTable("columns", StorageSystemColumns::create("columns")); - system_database.attachTable("functions", StorageSystemFunctions::create("functions")); - system_database.attachTable("events", StorageSystemEvents::create("events")); - system_database.attachTable("settings", StorageSystemSettings::create("settings")); - system_database.attachTable("merge_tree_settings", SystemMergeTreeSettings::create("merge_tree_settings")); - system_database.attachTable("build_options", StorageSystemBuildOptions::create("build_options")); - system_database.attachTable("formats", StorageSystemFormats::create("formats")); - system_database.attachTable("table_functions", StorageSystemTableFunctions::create("table_functions")); - system_database.attachTable("aggregate_function_combinators", StorageSystemAggregateFunctionCombinators::create("aggregate_function_combinators")); - system_database.attachTable("data_type_families", StorageSystemDataTypeFamilies::create("data_type_families")); - system_database.attachTable("collations", StorageSystemCollations::create("collations")); - system_database.attachTable("table_engines", StorageSystemTableEngines::create("table_engines")); - system_database.attachTable("contributors", StorageSystemContributors::create("contributors")); - system_database.attachTable("users", StorageSystemUsers::create("users")); - system_database.attachTable("roles", StorageSystemRoles::create("roles")); - system_database.attachTable("grants", StorageSystemGrants::create("grants")); - system_database.attachTable("role_grants", StorageSystemRoleGrants::create("role_grants")); - system_database.attachTable("current_roles", StorageSystemCurrentRoles::create("current_roles")); - system_database.attachTable("enabled_roles", StorageSystemEnabledRoles::create("enabled_roles")); - system_database.attachTable("settings_profiles", StorageSystemSettingsProfiles::create("settings_profiles")); - system_database.attachTable("settings_profile_elements", StorageSystemSettingsProfileElements::create("settings_profile_elements")); - system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); - system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); - system_database.attachTable("quota_limits", StorageSystemQuotaLimits::create("quota_limits")); - system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); - system_database.attachTable("quotas_usage", StorageSystemQuotasUsage::create("all_quotas_usage")); - system_database.attachTable("privileges", StorageSystemPrivileges::create("privileges")); + attach(system_database, "one"); + attach(system_database, "numbers", false); + attach(system_database, "numbers_mt", true); + attach(system_database, "zeros", false); + attach(system_database, "zeros_mt", true); + attach(system_database, "databases"); + attach(system_database, "tables"); + attach(system_database, "columns"); + attach(system_database, "functions"); + attach(system_database, "events"); + attach(system_database, "settings"); + attach(system_database, "merge_tree_settings"); + attach(system_database, "build_options"); + attach(system_database, "formats"); + attach(system_database, "table_functions"); + attach(system_database, "aggregate_function_combinators"); + attach(system_database, "data_type_families"); + attach(system_database, "collations"); + attach(system_database, "table_engines"); + attach(system_database, "contributors"); + attach(system_database, "users"); + attach(system_database, "roles"); + attach(system_database, "grants"); + attach(system_database, "role_grants"); + attach(system_database, "current_roles"); + attach(system_database, "enabled_roles"); + attach(system_database, "settings_profiles"); + attach(system_database, "settings_profile_elements"); + attach(system_database, "row_policies"); + attach(system_database, "quotas"); + attach(system_database, "quota_limits"); + attach(system_database, "quota_usage"); + attach(system_database, "quotas_usage"); + attach(system_database, "privileges"); + #if !defined(ARCADIA_BUILD) - system_database.attachTable("licenses", StorageSystemLicenses::create("licenses")); + attach(system_database, "licenses"); #endif #ifdef OS_LINUX - system_database.attachTable("stack_trace", StorageSystemStackTrace::create("stack_trace")); + attach(system_database, "stack_trace"); #endif } void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) { attachSystemTablesLocal(system_database); - system_database.attachTable("parts", StorageSystemParts::create("parts")); - system_database.attachTable("detached_parts", createDetachedPartsTable()); - system_database.attachTable("parts_columns", StorageSystemPartsColumns::create("parts_columns")); - system_database.attachTable("disks", StorageSystemDisks::create("disks")); - system_database.attachTable("storage_policies", StorageSystemStoragePolicies::create("storage_policies")); - system_database.attachTable("processes", StorageSystemProcesses::create("processes")); - system_database.attachTable("metrics", StorageSystemMetrics::create("metrics")); - system_database.attachTable("merges", StorageSystemMerges::create("merges")); - system_database.attachTable("mutations", StorageSystemMutations::create("mutations")); - system_database.attachTable("replicas", StorageSystemReplicas::create("replicas")); - system_database.attachTable("replication_queue", StorageSystemReplicationQueue::create("replication_queue")); - system_database.attachTable("distribution_queue", StorageSystemDistributionQueue::create("distribution_queue")); - system_database.attachTable("dictionaries", StorageSystemDictionaries::create("dictionaries")); - system_database.attachTable("models", StorageSystemModels::create("models")); - system_database.attachTable("clusters", StorageSystemClusters::create("clusters")); - system_database.attachTable("graphite_retentions", StorageSystemGraphite::create("graphite_retentions")); - system_database.attachTable("macros", StorageSystemMacros::create("macros")); + + attach(system_database, "parts"); + attach(system_database, "detached_parts"); + attach(system_database, "parts_columns"); + attach(system_database, "disks"); + attach(system_database, "storage_policies"); + attach(system_database, "processes"); + attach(system_database, "metrics"); + attach(system_database, "merges"); + attach(system_database, "mutations"); + attach(system_database, "replicas"); + attach(system_database, "replication_queue"); + attach(system_database, "distribution_queue"); + attach(system_database, "dictionaries"); + attach(system_database, "models"); + attach(system_database, "clusters"); + attach(system_database, "graphite_retentions"); + attach(system_database, "macros"); if (has_zookeeper) - system_database.attachTable("zookeeper", StorageSystemZooKeeper::create("zookeeper")); + attach(system_database, "zookeeper"); } void attachSystemTablesAsync(IDatabase & system_database, AsynchronousMetrics & async_metrics) { - system_database.attachTable("asynchronous_metrics", StorageSystemAsynchronousMetrics::create("asynchronous_metrics", async_metrics)); + attach(system_database, "asynchronous_metrics", async_metrics); } } diff --git a/src/Storages/System/attachSystemTablesImpl.h b/src/Storages/System/attachSystemTablesImpl.h new file mode 100644 index 00000000000..0dc34c3c116 --- /dev/null +++ b/src/Storages/System/attachSystemTablesImpl.h @@ -0,0 +1,27 @@ +#pragma once +#include + +namespace DB +{ + +template +void attach(IDatabase & system_database, const String & table_name, StorageArgs && ... args) +{ + if (system_database.getUUID() == UUIDHelpers::Nil) + { + /// Attach to Ordinary database + auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name); + system_database.attachTable(table_name, StorageT::create(table_id, std::forward(args)...)); + } + else + { + /// Attach to Atomic database + /// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages + /// and path is actually not used + auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4()); + String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid); + system_database.attachTable(table_name, StorageT::create(table_id, std::forward(args)...), path); + } +} + +} diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index e1c7e400249..89b69eb79e3 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -26,7 +26,6 @@ void rewriteEntityInAst(ASTPtr ast, const String & column_name, const Field & va /// Leave in the block only the rows that fit under the WHERE clause and the PREWHERE clause of the query. /// Only elements of the outer conjunction are considered, depending only on the columns present in the block. -/// Returns true if at least one row is discarded. void filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context); /// Extract from the input stream a set of `name` column values diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 12629628007..5ba14cdb8aa 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -49,6 +49,10 @@ def remove_control_characters(s): s = re.sub(r"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", s) return s +def get_db_engine(args): + if args.atomic_db_engine: + return " ENGINE=Atomic" + return "" def run_single_test(args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file): @@ -69,7 +73,7 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std database = 'test_{suffix}'.format(suffix=random_str()) clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("CREATE DATABASE " + database) + clickhouse_proc_create.communicate("CREATE DATABASE " + database + get_db_engine(args)) os.environ["CLICKHOUSE_DATABASE"] = database @@ -278,6 +282,10 @@ def run_tests_array(all_tests_with_params): if args.stop and ('Connection refused' in stderr or 'Attempt to read after eof' in stderr) and not 'Received exception from server' in stderr: SERVER_DIED = True + if os.path.isfile(stdout_file): + print(", result:\n") + print(open(stdout_file).read()) + elif stderr: failures += 1 failures_chain += 1 @@ -507,10 +515,10 @@ def main(args): if args.database and args.database != "test": clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database) + clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS " + args.database + get_db_engine(args)) clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test") + clickhouse_proc_create.communicate("CREATE DATABASE IF NOT EXISTS test" + get_db_engine(args)) def is_test_from_dir(suite_dir, case): case_file = os.path.join(suite_dir, case) @@ -722,6 +730,7 @@ if __name__ == '__main__': parser.add_argument('-r', '--server-check-retries', default=30, type=int, help='Num of tries to execute SELECT 1 before tests started') parser.add_argument('--skip-list-path', help="Path to skip-list file") parser.add_argument('--use-skip-list', action='store_true', default=False, help="Use skip list to skip tests if found") + parser.add_argument('--atomic-db-engine', action='store_true', help='Create databases with Atomic engine by default') parser.add_argument('--no-stateless', action='store_true', help='Disable all stateless tests') parser.add_argument('--no-stateful', action='store_true', help='Disable all stateful tests') diff --git a/tests/config/database_atomic_usersd.xml b/tests/config/database_atomic_usersd.xml index 9e749b231bd..201d476da24 100644 --- a/tests/config/database_atomic_usersd.xml +++ b/tests/config/database_atomic_usersd.xml @@ -2,7 +2,6 @@ Atomic - 1 0 diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index d00450bf245..bb54920593f 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -531,7 +531,6 @@ def test_start_stop_moves(start_cluster, name, engine): assert used_disks[0] == 'jbod1' node1.query("SYSTEM START MOVES {}".format(name)) - node1.query("SYSTEM START MERGES {}".format(name)) # wait sometime until background backoff finishes retry = 30 @@ -541,6 +540,8 @@ def test_start_stop_moves(start_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) i += 1 + node1.query("SYSTEM START MERGES {}".format(name)) + assert sum(1 for x in used_disks if x == 'jbod1') <= 2 # first (oldest) part moved to external diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index 10b4265b57f..aed00c905d7 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -13,7 +13,7 @@ $CLICKHOUSE_CLIENT -q "select name from system.table_functions format TSV;" > "$ # if you want long run use: env SQL_FUZZY_RUNS=100000 clickhouse-test sql_fuzzy for SQL_FUZZY_RUN in $(seq "${SQL_FUZZY_RUNS:=10}"); do - env SQL_FUZZY_RUN="$SQL_FUZZY_RUN" "$CURDIR"/00746_sql_fuzzy.pl | $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 + env SQL_FUZZY_RUN="$SQL_FUZZY_RUN" "$CURDIR"/00746_sql_fuzzy.pl | timeout 60 $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 if [[ $($CLICKHOUSE_CLIENT -q "SELECT 'Still alive'") != 'Still alive' ]]; then break fi diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index 57e92c97d11..028c75b8a6c 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -8,7 +8,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" $CLICKHOUSE_CLIENT --query="CREATE TABLE small_table (a UInt64 default 0, n UInt64) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY (a);" -$CLICKHOUSE_CLIENT --query="INSERT INTO small_table(n) SELECT * from system.numbers limit 100000;" +$CLICKHOUSE_CLIENT --query="INSERT INTO small_table (n) SELECT * from system.numbers limit 100000;" +$CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE small_table FINAL;" cached_query="SELECT count() FROM small_table where n > 0;" @@ -16,7 +17,6 @@ $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query="$cached_query" &> /dev/nu $CLICKHOUSE_CLIENT --use_uncompressed_cache=1 --query_id="test-query-uncompressed-cache" --query="$cached_query" &> /dev/null -sleep 1 $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" $CLICKHOUSE_CLIENT --query="SELECT ProfileEvents.Values[indexOf(ProfileEvents.Names, 'Seek')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ReadCompressedBytes')], ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UncompressedCacheHits')] AS hit FROM system.query_log WHERE (query_id = 'test-query-uncompressed-cache') AND (type = 2) AND event_date >= yesterday() ORDER BY event_time DESC LIMIT 1" diff --git a/tests/queries/0_stateless/00956_sensitive_data_masking.sh b/tests/queries/0_stateless/00956_sensitive_data_masking.sh index 337041be01b..9e3718b55f5 100755 --- a/tests/queries/0_stateless/00956_sensitive_data_masking.sh +++ b/tests/queries/0_stateless/00956_sensitive_data_masking.sh @@ -2,9 +2,6 @@ # Get all server logs export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="trace" -#export CLICKHOUSE_BINARY='../../../../build-vscode/Debug/programs/clickhouse' -#export CLICKHOUSE_PORT_TCP=59000 -#export CLICKHOUSE_CLIENT_BINARY='../../../../cmake-build-debug/programs/clickhouse client' CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh @@ -20,8 +17,8 @@ $CLICKHOUSE_CLIENT \ --query="SELECT 'find_me_TOPSECRET=TOPSECRET' FROM numbers(1) FORMAT Null" \ --log_queries=1 --ignore-error --multiquery >"$tmp_file" 2>&1 -grep 'find_me_\[hidden\]' "$tmp_file" >/dev/null || echo 'fail 1a' -grep 'TOPSECRET' "$tmp_file" && echo 'fail 1b' +grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 1a' +grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 1b' rm -f "$tmp_file" >/dev/null 2>&1 echo 2 @@ -31,8 +28,8 @@ echo "SELECT 'find_me_TOPSECRET=TOPSECRET' FRRRROM numbers" | ${CLICKHOUSE_CURL} #cat $tmp_file ## can't be checked on client side! -# grep 'find_me_\[hidden\]' $tmp_file >/dev/null || echo 'fail 2a' -grep 'TOPSECRET' "$tmp_file" && echo 'fail 2b' +# grep -F 'find_me_[hidden]' $tmp_file >/dev/null || echo 'fail 2a' +grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 2b' rm -f "$tmp_file" >/dev/null 2>&1 echo 3 @@ -41,8 +38,8 @@ $CLICKHOUSE_CLIENT \ --query="SELECT 'find_me_TOPSECRET=TOPSECRET' FROM non_existing_table FORMAT Null" \ --log_queries=1 --ignore-error --multiquery >"$tmp_file" 2>&1 -grep 'find_me_\[hidden\]' "$tmp_file" >/dev/null || echo 'fail 3a' -grep 'TOPSECRET' "$tmp_file" && echo 'fail 3b' +grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 3a' +grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 3b' rm -f "$tmp_file" >/dev/null 2>&1 echo 4 @@ -51,32 +48,32 @@ $CLICKHOUSE_CLIENT \ --query="SELECT 'find_me_TOPSECRET=TOPSECRET', intDiv( 100, number - 10) FROM numbers(11) FORMAT Null" \ --log_queries=1 --ignore-error --max_block_size=2 --multiquery >"$tmp_file" 2>&1 -grep 'find_me_\[hidden\]' "$tmp_file" >/dev/null || echo 'fail 4a' -grep 'TOPSECRET' "$tmp_file" && echo 'fail 4b' +grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 4a' +grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 4b' echo 5 # run in background rm -f "$tmp_file2" >/dev/null 2>&1 bash -c "$CLICKHOUSE_CLIENT \ - --query=\"select sleepEachRow(0.5) from numbers(4) where ignore('find_me_TOPSECRET=TOPSECRET')=0 and ignore('fwerkh_that_magic_string_make_me_unique') = 0 FORMAT Null\" \ + --query=\"select count() from system.numbers where ignore('find_me_TOPSECRET=TOPSECRET')=0 and ignore('fwerkh_that_magic_string_make_me_unique') = 0 FORMAT Null\" \ --log_queries=1 --ignore-error --multiquery >$tmp_file2 2>&1" & -sleep 0.1 - # $CLICKHOUSE_CLIENT --query='SHOW PROCESSLIST' rm -f "$tmp_file" >/dev/null 2>&1 echo '5.1' # check that executing query doesn't expose secrets in processlist $CLICKHOUSE_CLIENT --query="SHOW PROCESSLIST" --log_queries=0 >"$tmp_file" 2>&1 + +$CLICKHOUSE_CLIENT --query="KILL QUERY WHERE query LIKE '%fwerkh_that_magic_string_make_me_unique%'" > /dev/null 2>&1 wait grep 'TOPSECRET' "$tmp_file2" && echo 'fail 5d' rm -f "$tmp_file2" >/dev/null 2>&1 -grep 'fwerkh_that_magic_string_make_me_unique' "$tmp_file" >"$tmp_file2" || echo 'fail 5a' -grep 'find_me_\[hidden\]' "$tmp_file2" >/dev/null || echo 'fail 5b' -grep 'TOPSECRET' "$tmp_file" && echo 'fail 5c' +grep -F 'fwerkh_that_magic_string_make_me_unique' "$tmp_file" >"$tmp_file2" || echo 'fail 5a' +grep -F 'find_me_[hidden]' "$tmp_file2" >/dev/null || echo 'fail 5b' +grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 5c' # instead of disabling send_logs_level=trace (enabled globally for that test) - redir it's output to /dev/null @@ -95,7 +92,7 @@ echo 7 # and finally querylog $CLICKHOUSE_CLIENT \ --server_logs_file=/dev/null \ - --query="select * from system.query_log where event_time>now() - 10 and query like '%TOPSECRET%';" + --query="select * from system.query_log where event_time > now() - 10 and query like '%TOPSECRET%';" rm -f "$tmp_file" >/dev/null 2>&1 @@ -108,17 +105,16 @@ insert into sensitive select number as id, toDate('2019-01-01') as date, 'abcd' select * from sensitive WHERE value1 = 'find_me_TOPSECRET=TOPSECRET' FORMAT Null; drop table sensitive;" --log_queries=1 --ignore-error --multiquery >"$tmp_file" 2>&1 -grep 'find_me_\[hidden\]' "$tmp_file" >/dev/null || echo 'fail 8a' -grep 'TOPSECRET' "$tmp_file" && echo 'fail 8b' +grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 8a' +grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 8b' $CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" --server_logs_file=/dev/null -sleep 0.1; echo 9 $CLICKHOUSE_CLIENT \ --server_logs_file=/dev/null \ --query="SELECT if( count() > 0, 'text_log non empty', 'text_log empty') FROM system.text_log WHERE event_time>now() - 60 and message like '%find_me%'; - select * from system.text_log where event_time>now() - 60 and message like '%TOPSECRET=TOPSECRET%';" --ignore-error --multiquery + select * from system.text_log where event_time > now() - 60 and message like '%TOPSECRET=TOPSECRET%';" --ignore-error --multiquery echo 'finish' rm -f "$tmp_file" >/dev/null 2>&1 diff --git a/tests/queries/0_stateless/00981_topK_topKWeighted_long.sql b/tests/queries/0_stateless/00981_topK_topKWeighted_long.sql index c1f385fae80..5758f8ab8e5 100644 --- a/tests/queries/0_stateless/00981_topK_topKWeighted_long.sql +++ b/tests/queries/0_stateless/00981_topK_topKWeighted_long.sql @@ -2,8 +2,7 @@ DROP TABLE IF EXISTS topk; CREATE TABLE topk (val1 String, val2 UInt32) ENGINE = MergeTree ORDER BY val1; -INSERT INTO topk SELECT toString(number), number FROM numbers(3000000); -INSERT INTO topk SELECT toString(number % 10), 999999999 FROM numbers(1000000); +INSERT INTO topk WITH number % 7 = 0 AS frequent SELECT toString(frequent ? number % 10 : number), frequent ? 999999999 : number FROM numbers(4000000); SELECT arraySort(topK(10)(val1)) FROM topk; SELECT arraySort(topKWeighted(10)(val1, val2)) FROM topk; diff --git a/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql b/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql index bc0e8a30670..57ec4460e79 100644 --- a/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql +++ b/tests/queries/0_stateless/00988_constraints_replication_zookeeper.sql @@ -31,6 +31,11 @@ INSERT INTO replicated_constraints1 VALUES (10, 10); INSERT INTO replicated_constraints2 VALUES (10, 10); ALTER TABLE replicated_constraints1 ADD CONSTRAINT b_constraint CHECK b > 10; + +-- Otherwise "Metadata on replica is not up to date with common metadata in Zookeeper. Cannot alter." is possible. +SYSTEM SYNC REPLICA replicated_constraints1; +SYSTEM SYNC REPLICA replicated_constraints2; + ALTER TABLE replicated_constraints2 ADD CONSTRAINT a_constraint CHECK a < 10; SYSTEM SYNC REPLICA replicated_constraints1; @@ -40,4 +45,4 @@ INSERT INTO replicated_constraints1 VALUES (10, 11); -- { serverError 469 } INSERT INTO replicated_constraints2 VALUES (9, 10); -- { serverError 469 } DROP TABLE replicated_constraints1; -DROP TABLE replicated_constraints2; \ No newline at end of file +DROP TABLE replicated_constraints2; diff --git a/tests/queries/0_stateless/01042_h3_k_ring.reference b/tests/queries/0_stateless/01042_h3_k_ring.reference index 860ddac7547..b427bcf58ac 100644 --- a/tests/queries/0_stateless/01042_h3_k_ring.reference +++ b/tests/queries/0_stateless/01042_h3_k_ring.reference @@ -1,3 +1,2 @@ [581250224954015743,581259021047037951,581267817140060159,581276613233082367,581500913605148671,581518505791193087,581764796395814911] [581276613233082367] -[581276613233082367] diff --git a/tests/queries/0_stateless/01042_h3_k_ring.sql b/tests/queries/0_stateless/01042_h3_k_ring.sql index 30bd2f58801..35335df7ff2 100644 --- a/tests/queries/0_stateless/01042_h3_k_ring.sql +++ b/tests/queries/0_stateless/01042_h3_k_ring.sql @@ -1,3 +1,3 @@ SELECT arraySort(h3kRing(581276613233082367, 1)); SELECT h3kRing(581276613233082367, 0); -SELECT h3kRing(581276613233082367, -1); +SELECT h3kRing(581276613233082367, -1); -- { serverError 12 } diff --git a/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql b/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql index 730263a2b12..1987fffaa58 100644 --- a/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql +++ b/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql @@ -3,7 +3,7 @@ CREATE TABLE test Engine = MergeTree ORDER BY number AS SELECT number, toString( SELECT count() FROM test; -ALTER TABLE test detach partition tuple(); +ALTER TABLE test DETACH PARTITION tuple(); SELECT count() FROM test; diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index d35f40137e1..355e987a6de 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -17,6 +17,7 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10, 40)" for i in $(seq $REPLICAS); do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i" $CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_detach_$i" done diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference index d360a046958..933fda604d7 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference @@ -1,11 +1,11 @@ -CREATE TABLE test_01083.file\n(\n `n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') -CREATE TABLE test_01083.buffer\n(\n `n` Int8\n)\nENGINE = Buffer(\'test_01083\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) -CREATE TABLE test_01083.merge\n(\n `n` Int8\n)\nENGINE = Merge(\'test_01083\', \'distributed\') -CREATE TABLE test_01083.merge_tf AS merge(\'test_01083\', \'.*\') -CREATE TABLE test_01083.distributed\n(\n `n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'test_01083\', \'file\') -CREATE TABLE test_01083.distributed_tf AS cluster(\'test_shard_localhost\', \'test_01083\', \'buffer\') -CREATE TABLE test_01083.url\n(\n `n` UInt64,\n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+test_01083.merge+format+CSV\', \'CSV\') -CREATE TABLE test_01083.rich_syntax AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'test_01083\', \'view\'))) -CREATE VIEW test_01083.view\n(\n `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM test_01083.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM test_01083.file -CREATE DICTIONARY test_01083.dict\n(\n `n` UInt64,\n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\' DB \'test_01083\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) +CREATE TABLE default.file\n(\n `n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') +CREATE TABLE default.buffer\n(\n `n` Int8\n)\nENGINE = Buffer(\'default\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) +CREATE TABLE default.merge\n(\n `n` Int8\n)\nENGINE = Merge(\'default\', \'distributed\') +CREATE TABLE default.merge_tf AS merge(\'default\', \'.*\') +CREATE TABLE default.distributed\n(\n `n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'default\', \'file\') +CREATE TABLE default.distributed_tf AS cluster(\'test_shard_localhost\', \'default\', \'buffer\') +CREATE TABLE default.url\n(\n `n` UInt64,\n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+default.merge+format+CSV\', \'CSV\') +CREATE TABLE default.rich_syntax AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'default\', \'view\'))) +CREATE VIEW default.view\n(\n `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM default.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM default.file +CREATE DICTIONARY default.dict\n(\n `n` UInt64,\n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) 16 diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 8e5f5a148a5..22aa4434f19 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -1,7 +1,3 @@ -DROP DATABASE IF EXISTS test_01083; -CREATE DATABASE test_01083; -USE test_01083; - CREATE TABLE file (n Int8) ENGINE = File(upper('tsv') || 'WithNames' || 'AndTypes'); CREATE TABLE buffer (n Int8) ENGINE = Buffer(currentDatabase(), file, 16, 10, 200, 10000, 1000000, 10000000, 1000000000); CREATE TABLE merge (n Int8) ENGINE = Merge('', lower('DISTRIBUTED')); @@ -28,7 +24,7 @@ CREATE VIEW view AS SELECT toInt64(n) as n FROM (SELECT toString(n) as n from me SELECT nonexistentsomething; -- { serverError 47 } CREATE DICTIONARY dict (n UInt64, col String DEFAULT '42') PRIMARY KEY n -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9440 SECURE 1 USER 'default' TABLE 'url' DB 'test_01083')) LIFETIME(1) LAYOUT(CACHE(SIZE_IN_CELLS 1)); +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9440 SECURE 1 USER 'default' TABLE 'url')) LIFETIME(1) LAYOUT(CACHE(SIZE_IN_CELLS 1)); -- dict --> url --> merge |-> distributed -> file (1) -- |-> distributed_tf -> buffer -> file (1) @@ -72,5 +68,3 @@ INSERT INTO buffer VALUES (1); -- | |-> file (1) -- |-> remote(127.0.0.2) --> ... SELECT sum(n) from rich_syntax; - -DROP DATABASE test_01083; diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 18d74a1817c..9f4bfb2c436 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01107" -$CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 -q "CREATE DATABASE test_01107 ENGINE=Atomic" +$CLICKHOUSE_CLIENT -q "CREATE DATABASE test_01107 ENGINE=Atomic" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01107.mt (n UInt64) ENGINE=MergeTree() ORDER BY tuple()" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(3) FROM numbers(5)" & @@ -18,7 +18,7 @@ wait $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" $CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" -$CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 -q "ATTACH DATABASE test_01107" +$CLICKHOUSE_CLIENT -q "ATTACH DATABASE test_01107" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" & diff --git a/tests/queries/0_stateless/01109_exchange_tables.sql b/tests/queries/0_stateless/01109_exchange_tables.sql index 7125bfea851..7fbb36e8ce9 100644 --- a/tests/queries/0_stateless/01109_exchange_tables.sql +++ b/tests/queries/0_stateless/01109_exchange_tables.sql @@ -1,5 +1,4 @@ DROP DATABASE IF EXISTS test_01109; -SET allow_experimental_database_atomic=1; CREATE DATABASE test_01109 ENGINE=Atomic; USE test_01109; diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index c7862bf5314..d23be44e784 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -8,8 +8,8 @@ $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01114_2" $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01114_3" -$CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 -q "CREATE DATABASE test_01114_1 ENGINE=Atomic" -$CLICKHOUSE_CLIENT --default_database_engine=Atomic --allow_experimental_database_atomic=1 -q "CREATE DATABASE test_01114_2" +$CLICKHOUSE_CLIENT -q "CREATE DATABASE test_01114_1 ENGINE=Atomic" +$CLICKHOUSE_CLIENT --default_database_engine=Atomic -q "CREATE DATABASE test_01114_2" $CLICKHOUSE_CLIENT --default_database_engine=Ordinary -q "CREATE DATABASE test_01114_3" $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=0 -q "SHOW CREATE DATABASE test_01114_1" diff --git a/tests/queries/0_stateless/01192_rename_database.sh b/tests/queries/0_stateless/01192_rename_database.sh index 3d4223d09d5..c97ec14335c 100755 --- a/tests/queries/0_stateless/01192_rename_database.sh +++ b/tests/queries/0_stateless/01192_rename_database.sh @@ -10,7 +10,7 @@ $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01192_renamed" $CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01192_atomic" $CLICKHOUSE_CLIENT --default_database_engine=Ordinary -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001'" 2>&1| grep -F "does not support" > /dev/null && echo "ok" -$CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 --default_database_engine=Atomic -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001'" +$CLICKHOUSE_CLIENT --default_database_engine=Atomic -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001'" # 2. check metadata $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE DATABASE test_01192" @@ -40,7 +40,7 @@ $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW test_01192.mv TO test_01192.rmt $CLICKHOUSE_CLIENT -q "INSERT INTO test_01192.mt SELECT number FROM numbers(10)" && echo "inserted" -$CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 --default_database_engine=Atomic -q "CREATE DATABASE test_01192_atomic" +$CLICKHOUSE_CLIENT --default_database_engine=Atomic -q "CREATE DATABASE test_01192_atomic" $CLICKHOUSE_CLIENT -q "DROP DATABASE test_01192_renamed" # it's blocking $CLICKHOUSE_CLIENT -q "RENAME TABLE test_01192.mt TO test_01192_atomic.mt, test_01192.rmt TO test_01192_atomic.rmt, test_01192.mv TO test_01192_atomic.mv" && echo "renamed" diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.reference b/tests/queries/0_stateless/01231_log_queries_min_type.reference index a358d022033..61f4c09dd20 100644 --- a/tests/queries/0_stateless/01231_log_queries_min_type.reference +++ b/tests/queries/0_stateless/01231_log_queries_min_type.reference @@ -1,5 +1,6 @@ 01231_log_queries_min_type/QUERY_START 2 01231_log_queries_min_type/EXCEPTION_BEFORE_START -2 -3 +0 +1 +1 diff --git a/tests/queries/0_stateless/01231_log_queries_min_type.sql b/tests/queries/0_stateless/01231_log_queries_min_type.sql index f2229c94a8a..bfeeca96d4b 100644 --- a/tests/queries/0_stateless/01231_log_queries_min_type.sql +++ b/tests/queries/0_stateless/01231_log_queries_min_type.sql @@ -2,14 +2,25 @@ set log_queries=1; select '01231_log_queries_min_type/QUERY_START'; system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +select count() from system.query_log where query like '%01231_log_queries_min_type/QUERY_START%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; set log_queries_min_type='EXCEPTION_BEFORE_START'; select '01231_log_queries_min_type/EXCEPTION_BEFORE_START'; system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +select count() from system.query_log where query like '%01231_log_queries_min_type/EXCEPTION_BEFORE_START%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +set max_rows_to_read='100K'; set log_queries_min_type='EXCEPTION_WHILE_PROCESSING'; -select '01231_log_queries_min_type/', max(number) from system.numbers limit 1e6 settings max_rows_to_read='100K'; -- { serverError 158; } +select '01231_log_queries_min_type/EXCEPTION_WHILE_PROCESSING', max(number) from system.numbers limit 1e6; -- { serverError 158; } system flush logs; -select count() from system.query_log where query like '%01231_log_queries_min_type/%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute; +select count() from system.query_log where query like '%01231_log_queries_min_type/EXCEPTION_WHILE_PROCESSING%' and query not like '%system.query_log%' and event_date = today() and event_time >= now() - interval 1 minute and type = 'ExceptionWhileProcessing'; + +select '01231_log_queries_min_type w/ Settings/EXCEPTION_WHILE_PROCESSING', max(number) from system.numbers limit 1e6; -- { serverError 158; } +system flush logs; +select count() from system.query_log where + query like '%01231_log_queries_min_type w/ Settings/EXCEPTION_WHILE_PROCESSING%' and + query not like '%system.query_log%' and + event_date = today() and + event_time >= now() - interval 1 minute and + type = 'ExceptionWhileProcessing' and + has(Settings.Names, 'max_rows_to_read'); diff --git a/tests/queries/0_stateless/01249_flush_interactive.sh b/tests/queries/0_stateless/01249_flush_interactive.sh index 01b91cbace8..6049c70b9df 100755 --- a/tests/queries/0_stateless/01249_flush_interactive.sh +++ b/tests/queries/0_stateless/01249_flush_interactive.sh @@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # unless the my-program will try to output a thousand more lines overflowing pipe buffer and terminating with Broken Pipe. # But if my program just output 5 (or slightly more) lines and hang up, the pipeline is not terminated. -timeout 1 ${CLICKHOUSE_LOCAL} --max_execution_time 10 --query "SELECT DISTINCT number % 5 FROM system.numbers" ||: +timeout 5 ${CLICKHOUSE_LOCAL} --max_execution_time 10 --query "SELECT DISTINCT number % 5 FROM system.numbers" ||: echo '---' -timeout 1 ${CLICKHOUSE_CURL} -sS --no-buffer "${CLICKHOUSE_URL}&max_execution_time=10" --data-binary "SELECT DISTINCT number % 5 FROM system.numbers" ||: +timeout 5 ${CLICKHOUSE_CURL} -sS --no-buffer "${CLICKHOUSE_URL}&max_execution_time=10" --data-binary "SELECT DISTINCT number % 5 FROM system.numbers" ||: echo '---' diff --git a/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.reference b/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.reference new file mode 100644 index 00000000000..bdd25a5ec95 --- /dev/null +++ b/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.reference @@ -0,0 +1,9 @@ +CSVWithNames +bar +TSVWithNames +117 + +TSV +117 +key\r +bar diff --git a/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh b/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh new file mode 100755 index 00000000000..5a1edb42a06 --- /dev/null +++ b/tests/queries/0_stateless/01406_carriage_return_in_tsv_csv.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2028 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +echo 'CSVWithNames' +echo -n $'key\r\nbar\r\n' | ${CLICKHOUSE_LOCAL} --input-format CSVWithNames -S 'key String' -q 'select * from table' + +echo 'TSVWithNames' +echo -n $'key\r\nbar\r\n' | ${CLICKHOUSE_LOCAL} --input-format TSVWithNames -S 'key String' -q 'select * from table' >& /dev/null +echo $? +echo -n $'key\\r\nbar\n' | ${CLICKHOUSE_LOCAL} --input_format_skip_unknown_fields=1 --input-format TSVWithNames -S 'key String' -q 'select * from table' + +echo 'TSV' +echo -n $'key\r\nbar\r\n' | ${CLICKHOUSE_LOCAL} --input-format TSV -S 'key String' -q 'select * from table' >& /dev/null +echo $? +echo -n $'key\\r\nbar\n' | ${CLICKHOUSE_LOCAL} --input-format TSV -S 'key String' -q 'select * from table' diff --git a/tests/queries/0_stateless/01408_range_overflow.reference b/tests/queries/0_stateless/01408_range_overflow.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01408_range_overflow.sql b/tests/queries/0_stateless/01408_range_overflow.sql new file mode 100644 index 00000000000..1640798999c --- /dev/null +++ b/tests/queries/0_stateless/01408_range_overflow.sql @@ -0,0 +1,12 @@ +-- executeGeneric() +SELECT range(1025, 1048576 + 9223372036854775807, 9223372036854775807); -- { serverError 69; } +SELECT range(1025, 1048576 + (9223372036854775807 AS i), i); -- { serverError 69; } + +-- executeConstStep() +SELECT range(number, 1048576 + 9223372036854775807, 9223372036854775807) FROM system.numbers LIMIT 1 OFFSET 1025; -- { serverError 69; } + +-- executeConstStartStep() +SELECT range(1025, number + 9223372036854775807, 9223372036854775807) FROM system.numbers LIMIT 1 OFFSET 1048576; -- { serverError 69; } + +-- executeConstStart() +SELECT range(1025, 1048576 + 9223372036854775807, number + 9223372036854775807) FROM system.numbers LIMIT 1; -- { serverError 69; } diff --git a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh index 3e17ea205d5..1f14a891dd4 100755 --- a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh +++ b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh @@ -29,7 +29,7 @@ query_result=$($CLICKHOUSE_CLIENT --query="$check_query" 2>&1) while [ "$query_result" != "1" ] do query_result=$($CLICKHOUSE_CLIENT --query="$check_query" 2>&1) - sleep 0.5 + sleep 0.1 done $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE table='replicated_mutation_table' and database='$CLICKHOUSE_DATABASE' and mutation_id='0000000000'" &> /dev/null @@ -42,9 +42,10 @@ done wait + $CLICKHOUSE_CLIENT --query "ALTER TABLE replicated_mutation_table MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync = 2" 2>&1 | grep -o "Cannot parse string 'Hello' as UInt64" | head -n 1 & -check_query="SELECT count() FROM system.mutations WHERE table='replicated_mutation_table' and database='$CLICKHOUSE_DATABASE' and mutation_id='0000000001'" +check_query="SELECT type = 'UInt64' FROM system.columns WHERE table='replicated_mutation_table' and database='$CLICKHOUSE_DATABASE' and name='value'" query_result=$($CLICKHOUSE_CLIENT --query="$check_query" 2>&1) @@ -56,6 +57,9 @@ done wait + +check_query="SELECT count() FROM system.mutations WHERE table='replicated_mutation_table' and database='$CLICKHOUSE_DATABASE' and mutation_id='0000000001'" + $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE table='replicated_mutation_table' and database='$CLICKHOUSE_DATABASE' AND mutation_id='0000000001'" &> /dev/null while [ "$query_result" != "0" ] diff --git a/tests/queries/0_stateless/01436_storage_merge_with_join_push_down.reference b/tests/queries/0_stateless/01436_storage_merge_with_join_push_down.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/01436_storage_merge_with_join_push_down.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/01436_storage_merge_with_join_push_down.sql b/tests/queries/0_stateless/01436_storage_merge_with_join_push_down.sql new file mode 100644 index 00000000000..a3c598c6d83 --- /dev/null +++ b/tests/queries/0_stateless/01436_storage_merge_with_join_push_down.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS test1; +DROP TABLE IF EXISTS test1_distributed; +DROP TABLE IF EXISTS test_merge; + +SET enable_optimize_predicate_expression = 1; + +CREATE TABLE test1 (id Int64, name String) ENGINE MergeTree PARTITION BY (id) ORDER BY (id); +CREATE TABLE test1_distributed AS test1 ENGINE = Distributed(test_cluster_two_shards_localhost, default, test1); +CREATE TABLE test_merge AS test1 ENGINE = Merge('default', 'test1_distributed'); + +SELECT count() FROM test_merge +JOIN (SELECT 'anystring' AS name) AS n +USING name +WHERE id = 1; + +DROP TABLE test1; +DROP TABLE test_merge; + + +CREATE TABLE test1 (id Int64, name String) ENGINE MergeTree PARTITION BY (id) ORDER BY (id); +CREATE TABLE test_merge AS test1 ENGINE = Merge('default', 'test1'); + +SELECT count() FROM test_merge +JOIN (SELECT 'anystring' AS name) AS n +USING name +WHERE id = 1; + +DROP TABLE test1; +DROP TABLE test_merge; +DROP TABLE test1_distributed; diff --git a/tests/queries/0_stateless/01441_array_combinator.reference b/tests/queries/0_stateless/01441_array_combinator.reference new file mode 100644 index 00000000000..7f1bc308d22 --- /dev/null +++ b/tests/queries/0_stateless/01441_array_combinator.reference @@ -0,0 +1,10 @@ +0 0 +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 diff --git a/tests/queries/0_stateless/01441_array_combinator.sql b/tests/queries/0_stateless/01441_array_combinator.sql new file mode 100644 index 00000000000..68fd050940d --- /dev/null +++ b/tests/queries/0_stateless/01441_array_combinator.sql @@ -0,0 +1 @@ +SELECT number % 100 AS k, sumArray(emptyArrayUInt8()) AS v FROM numbers(10) GROUP BY k; diff --git a/tests/queries/0_stateless/01442_h3kring_range_check.reference b/tests/queries/0_stateless/01442_h3kring_range_check.reference new file mode 100644 index 00000000000..9f54fe3133b --- /dev/null +++ b/tests/queries/0_stateless/01442_h3kring_range_check.reference @@ -0,0 +1 @@ +122 diff --git a/tests/queries/0_stateless/01442_h3kring_range_check.sql b/tests/queries/0_stateless/01442_h3kring_range_check.sql new file mode 100644 index 00000000000..e580beb17a3 --- /dev/null +++ b/tests/queries/0_stateless/01442_h3kring_range_check.sql @@ -0,0 +1,4 @@ +SELECT h3kRing(581276613233082367, 65535); -- { serverError 12 } +SELECT h3kRing(581276613233082367, -1); -- { serverError 12 } +SELECT length(h3kRing(111111111111, 1000)); +SELECT h3kRing(581276613233082367, nan); -- { serverError 43 } diff --git a/tests/queries/0_stateless/01442_merge_detach_attach.reference b/tests/queries/0_stateless/01442_merge_detach_attach.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01442_merge_detach_attach.sh b/tests/queries/0_stateless/01442_merge_detach_attach.sh new file mode 100755 index 00000000000..4577d805da5 --- /dev/null +++ b/tests/queries/0_stateless/01442_merge_detach_attach.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t (x Int8) ENGINE = MergeTree ORDER BY tuple()" + +for _ in {1..100}; do + ${CLICKHOUSE_CLIENT} --query="INSERT INTO t VALUES (0)" + ${CLICKHOUSE_CLIENT} --query="INSERT INTO t VALUES (0)" + ${CLICKHOUSE_CLIENT} --query="OPTIMIZE TABLE t FINAL" 2>/dev/null & + ${CLICKHOUSE_CLIENT} --query="ALTER TABLE t DETACH PARTITION tuple()" + ${CLICKHOUSE_CLIENT} --query="SELECT count() FROM t HAVING count() > 0" +done + +wait diff --git a/tests/queries/0_stateless/01443_merge_truncate.reference b/tests/queries/0_stateless/01443_merge_truncate.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01443_merge_truncate.sh b/tests/queries/0_stateless/01443_merge_truncate.sh new file mode 100755 index 00000000000..5a0d1188ab6 --- /dev/null +++ b/tests/queries/0_stateless/01443_merge_truncate.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t (x Int8) ENGINE = MergeTree ORDER BY tuple()" + +for _ in {1..100}; do + ${CLICKHOUSE_CLIENT} --query="INSERT INTO t VALUES (0)" + ${CLICKHOUSE_CLIENT} --query="INSERT INTO t VALUES (0)" + ${CLICKHOUSE_CLIENT} --query="OPTIMIZE TABLE t FINAL" 2>/dev/null & + ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE t" + ${CLICKHOUSE_CLIENT} --query="SELECT count() FROM t HAVING count() > 0" +done + +wait diff --git a/tests/queries/0_stateless/01450_set_null_const.reference b/tests/queries/0_stateless/01450_set_null_const.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01450_set_null_const.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01450_set_null_const.sql b/tests/queries/0_stateless/01450_set_null_const.sql new file mode 100644 index 00000000000..c47176a880f --- /dev/null +++ b/tests/queries/0_stateless/01450_set_null_const.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS test_mtree; + +CREATE TABLE test_mtree (`x` String, INDEX idx x TYPE set(10) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO test_mtree VALUES ('Hello, world'); +SELECT count() FROM test_mtree WHERE x = NULL; + +DROP TABLE test_mtree; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index c40da4c8e50..2b4d38433a7 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -12,7 +12,8 @@ "01103_check_cpu_instructions_at_startup", "01098_temporary_and_external_tables", "00152_insert_different_granularity", - "00151_replace_partition_with_different_granularity" + "00151_replace_partition_with_different_granularity", + "01193_metadata_loading" ], "address-sanitizer": [ "00281", @@ -21,7 +22,8 @@ "query_profiler", "memory_profiler", "odbc_roundtrip", - "01103_check_cpu_instructions_at_startup" + "01103_check_cpu_instructions_at_startup", + "01193_metadata_loading" ], "ub-sanitizer": [ "00281", @@ -30,7 +32,8 @@ "query_profiler", "memory_profiler", "01103_check_cpu_instructions_at_startup", - "00900_orc_load" + "00900_orc_load", + "01193_metadata_loading" ], "memory-sanitizer": [ "00281", @@ -41,7 +44,8 @@ "01103_check_cpu_instructions_at_startup", "01086_odbc_roundtrip", "00877_memory_limit_for_new_delete", - "01114_mysql_database_engine_segfault" + "01114_mysql_database_engine_segfault", + "01193_metadata_loading" ], "debug-build": [ "00281", @@ -55,7 +59,8 @@ "01200_mutations_memory_consumption", "01103_check_cpu_instructions_at_startup", "01037_polygon_dicts_", - "hyperscan" + "hyperscan", + "01193_metadata_loading" ], "unbundled-build": [ "00429", @@ -82,7 +87,8 @@ "01099_parallel_distributed_insert_select", "01300_client_save_history_when_terminated", "orc_output", - "01370_client_autocomplete_word_break_characters" + "01370_client_autocomplete_word_break_characters", + "01193_metadata_loading" ], "release-build": [ "avx2" diff --git a/tests/testflows/example/docker-compose/clickhouse-service.yml b/tests/testflows/example/docker-compose/clickhouse-service.yml index 36f13e40c5c..2a56876c72e 100644 --- a/tests/testflows/example/docker-compose/clickhouse-service.yml +++ b/tests/testflows/example/docker-compose/clickhouse-service.yml @@ -18,10 +18,10 @@ services: entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" healthcheck: test: clickhouse client --query='select 1' - interval: 3s - timeout: 5s - retries: 100 - start_period: 2s + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s cap_add: - SYS_PTRACE security_opt: diff --git a/tests/testflows/example/docker-compose/zookeeper-service.yml b/tests/testflows/example/docker-compose/zookeeper-service.yml index f3df33358be..6691a2df31c 100644 --- a/tests/testflows/example/docker-compose/zookeeper-service.yml +++ b/tests/testflows/example/docker-compose/zookeeper-service.yml @@ -10,9 +10,9 @@ services: ZOO_MY_ID: 1 healthcheck: test: echo stat | nc localhost 2181 - interval: 3s - timeout: 2s - retries: 5 - start_period: 2s + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s security_opt: - label:disable diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 660e831ffb8..e7fbc6bde01 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -92,7 +92,7 @@ class ClickHouseNode(Node): name, value = setting command += f" --{name} \"{value}\"" description = f""" - echo -e \"{sql[:100]}...\" > {query.name} + echo -e \"{sql[:100]}...\" > {query.name} {command} """ with step("executing command", description=description) if steps else NullStep(): @@ -128,12 +128,12 @@ class ClickHouseNode(Node): class Cluster(object): """Simple object around docker-compose cluster. """ - def __init__(self, local=False, - clickhouse_binary_path=None, configs_dir=None, + def __init__(self, local=False, + clickhouse_binary_path=None, configs_dir=None, nodes=None, - docker_compose="docker-compose", docker_compose_project_dir=None, + docker_compose="docker-compose", docker_compose_project_dir=None, docker_compose_file="docker-compose.yml"): - + self._bash = {} self.clickhouse_binary_path = clickhouse_binary_path self.configs_dir = configs_dir @@ -159,7 +159,7 @@ class Cluster(object): if os.path.exists(caller_project_dir): docker_compose_project_dir = caller_project_dir - docker_compose_file_path = os.path.join(docker_compose_project_dir or "", docker_compose_file) + docker_compose_file_path = os.path.join(docker_compose_project_dir or "", docker_compose_file) if not os.path.exists(docker_compose_file_path): raise TypeError("docker compose file '{docker_compose_file_path}' does not exist") @@ -237,7 +237,7 @@ class Cluster(object): finally: return self.command(None, f"{self.docker_compose} down", timeout=timeout) - def up(self): + def up(self, timeout=30*60): if self.local: with Given("I am running in local mode"): with Then("check --clickhouse-binary-path is specified"): @@ -245,20 +245,26 @@ class Cluster(object): with And("path should exist"): assert os.path.exists(self.clickhouse_binary_path) - os.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = self.clickhouse_binary_path - os.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = os.path.join( - os.path.dirname(self.clickhouse_binary_path), "clickhouse-odbc-bridge") - os.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir + with And("I set all the necessary environment variables"): + os.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = self.clickhouse_binary_path + os.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = os.path.join( + os.path.dirname(self.clickhouse_binary_path), "clickhouse-odbc-bridge") + os.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir - with Given("docker-compose"): + with And("I list environment variables to show their values"): self.command(None, "env | grep CLICKHOUSE") - cmd = self.command(None, f'{self.docker_compose} up -d 2>&1 | tee', timeout=30 * 60) - else: - with Given("docker-compose"): - cmd = self.command(None, f'{self.docker_compose} up -d --no-recreate 2>&1 | tee') + + with Given("docker-compose"): + with By("pulling images for all the services"): + self.command(None, f'{self.docker_compose} pull 2>&1 | tee', exitcode=0, timeout=timeout) + with And("executing docker-compose down just in case it is up"): + self.command(None, f'{self.docker_compose} down 2>&1 | tee', exitcode=0, timeout=timeout) + with And("executing docker-compose up"): + cmd = self.command(None, f'{self.docker_compose} up -d 2>&1 | tee', timeout=timeout) with Then("check there are no unhealthy containers"): if "is unhealthy" in cmd.output: + self.command(None, f'{self.docker_compose} ps | tee') self.command(None, f'{self.docker_compose} logs | tee') fail("found unhealthy containers") diff --git a/tests/testflows/ldap/configs/CA/dhparam.pem b/tests/testflows/ldap/configs/CA/dhparam.pem index 0a96faffd62..554d75696ee 100644 --- a/tests/testflows/ldap/configs/CA/dhparam.pem +++ b/tests/testflows/ldap/configs/CA/dhparam.pem @@ -1,5 +1,8 @@ -----BEGIN DH PARAMETERS----- -MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV -AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 -FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +MIIBCAKCAQEA1iatTn4jdw1WIu09qeLj8OEeLhzG/w2lI4RUeJT9nU+WTwegpvLN +/MvrIMIKHRmItyxgraYFau2moC7RKm7OKLmFt6e34QeMvM1vXpuwQav6mfp8GsYL +mEIw5riFcB73E32NN3g7qmfmurkTF28BohmqhuQp2et7FNoGBKQ6ePZzGHWil3yG +nEnCwyK0o3eP2IEytx2N50uUWVdfg3MN34L3wqpUivArrjBkoMpqm3/V3wdfoYG9 +ZQkH0gIxT/2FIixCLGlfBsJ1qA/Apz1BJZbGqVu5M5iiQmq+LWN5JLS3xYai4wJL +rIY8DhjbciSNVWkwTJHzaLwIQa9a6p6mUwIBAg== -----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/configs/ldap4/certs/ca.crt b/tests/testflows/ldap/configs/ldap4/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/configs/ldap4/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/configs/ldap4/certs/dhparam.pem b/tests/testflows/ldap/configs/ldap4/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/configs/ldap4/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.crt b/tests/testflows/ldap/configs/ldap4/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/configs/ldap4/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.csr b/tests/testflows/ldap/configs/ldap4/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/configs/ldap4/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.key b/tests/testflows/ldap/configs/ldap4/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/configs/ldap4/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/docker-compose/clickhouse-service.yml b/tests/testflows/ldap/docker-compose/clickhouse-service.yml index 12553d7948d..2a56876c72e 100644 --- a/tests/testflows/ldap/docker-compose/clickhouse-service.yml +++ b/tests/testflows/ldap/docker-compose/clickhouse-service.yml @@ -18,10 +18,10 @@ services: entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" healthcheck: test: clickhouse client --query='select 1' - interval: 3s + interval: 10s timeout: 10s - retries: 100 - start_period: 30s + retries: 3 + start_period: 300s cap_add: - SYS_PTRACE security_opt: diff --git a/tests/testflows/ldap/docker-compose/docker-compose.yml b/tests/testflows/ldap/docker-compose/docker-compose.yml index c788e089b4f..c8ff683df58 100644 --- a/tests/testflows/ldap/docker-compose/docker-compose.yml +++ b/tests/testflows/ldap/docker-compose/docker-compose.yml @@ -57,9 +57,6 @@ services: LDAP_TLS_ENFORCE: "false" LDAP_TLS_VERIFY_CLIENT: "never" LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" - healthcheck: - # May need some time to generate DH parameters... - start_period: 300s volumes: - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" diff --git a/tests/testflows/ldap/docker-compose/openldap-service.yml b/tests/testflows/ldap/docker-compose/openldap-service.yml index d642c535f30..139907c513c 100644 --- a/tests/testflows/ldap/docker-compose/openldap-service.yml +++ b/tests/testflows/ldap/docker-compose/openldap-service.yml @@ -14,10 +14,10 @@ services: - "636" healthcheck: test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin - interval: 3s + interval: 10s timeout: 10s - retries: 5 - start_period: 30s + retries: 3 + start_period: 300s security_opt: - label:disable @@ -31,10 +31,10 @@ services: - "8080:80" healthcheck: test: echo 1 - interval: 3s - timeout: 2s - retries: 5 - start_period: 2s + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s security_opt: - label:disable diff --git a/tests/testflows/ldap/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/docker-compose/zookeeper-service.yml index f3df33358be..6691a2df31c 100644 --- a/tests/testflows/ldap/docker-compose/zookeeper-service.yml +++ b/tests/testflows/ldap/docker-compose/zookeeper-service.yml @@ -10,9 +10,9 @@ services: ZOO_MY_ID: 1 healthcheck: test: echo stat | nc localhost 2181 - interval: 3s - timeout: 2s - retries: 5 - start_period: 2s + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s security_opt: - label:disable diff --git a/tests/testflows/rbac/docker-compose/clickhouse-service.yml b/tests/testflows/rbac/docker-compose/clickhouse-service.yml index a41f0da2839..2d79443dcbb 100644 --- a/tests/testflows/rbac/docker-compose/clickhouse-service.yml +++ b/tests/testflows/rbac/docker-compose/clickhouse-service.yml @@ -18,10 +18,10 @@ services: entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" healthcheck: test: clickhouse client --query='select 1' - interval: 3s - timeout: 2s - retries: 100 - start_period: 2s + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s cap_add: - SYS_PTRACE security_opt: diff --git a/tests/testflows/rbac/docker-compose/zookeeper-service.yml b/tests/testflows/rbac/docker-compose/zookeeper-service.yml index f3df33358be..6691a2df31c 100644 --- a/tests/testflows/rbac/docker-compose/zookeeper-service.yml +++ b/tests/testflows/rbac/docker-compose/zookeeper-service.yml @@ -10,9 +10,9 @@ services: ZOO_MY_ID: 1 healthcheck: test: echo stat | nc localhost 2181 - interval: 3s - timeout: 2s - retries: 5 - start_period: 2s + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s security_opt: - label:disable diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 939eb3ecca8..83284125681 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -68,3 +68,16 @@ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | gre # There shouldn't be any code snippets under GPL or LGPL find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL" + +# Check for typos in code +CURDIR=$(dirname "${BASH_SOURCE[0]}") +"${CURDIR}"/check-typos + +# Check sh tests with Shellcheck +(cd $ROOT_PATH/tests/queries/0_stateless/ && shellcheck --check-sourced --external-sources --severity info --exclude SC1071,SC2086 *.sh ../1_stateful/*.sh) + +# There shouldn't be any docker containers outside docker directory +find $ROOT_PATH -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:" + +# There shouldn't be any docker compose files outside docker directory +#find $ROOT_PATH -not -path $ROOT_PATH'/tests/testflows*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name '*compose*.yml' -type f 2>/dev/null | xargs --no-run-if-empty grep -l "version:" | xargs --no-run-if-empty -n1 echo "Please move docker compose to docker directory:" diff --git a/utils/check-style/check-typos b/utils/check-style/check-typos new file mode 100755 index 00000000000..14deb039059 --- /dev/null +++ b/utils/check-style/check-typos @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Check for typos in code. + +ROOT_PATH=$(git rev-parse --show-toplevel) +CURDIR=$(dirname "${BASH_SOURCE[0]}") + +codespell \ + --skip '*generated*,*gperf*,*.bin,*.mrk*,*.idx,checksums.txt,*.dat,*.pyc,*.kate-swp' \ + --ignore-words "${CURDIR}/codespell-ignore-words.list" \ + --exclude-file "${CURDIR}/codespell-ignore-lines.list" \ + --quiet-level 2 \ + "$ROOT_PATH"/{src,base,programs,utils} \ + $@ | grep -P '.' \ + && echo -e "\nFound some typos in code.\nSee the files utils/check-style/codespell* if you want to add an exception." diff --git a/utils/check-style/codespell-ignore-lines.list b/utils/check-style/codespell-ignore-lines.list new file mode 100644 index 00000000000..7c2959e9468 --- /dev/null +++ b/utils/check-style/codespell-ignore-lines.list @@ -0,0 +1,6 @@ +/// https://groups.google.com/forum/#!searchin/kona-dev/variadic$20macro%7Csort:date/kona-dev/XMA-lDOqtlI/GCzdfZsD41sJ +Jo-Philipp Wich +The TRE regular expression implementation (src/regex/reg* and +src/regex/tre*) is Copyright © 2001-2008 Ville Laurikari and licensed + pullRequests(first: {min_page_size} baseRefName: "{base}" headRefName: "{head}") {{ + uint64_t time_to_wait = nanoseconds * timebase_info.denom / timebase_info.numer; diff --git a/utils/check-style/codespell-ignore-words.list b/utils/check-style/codespell-ignore-words.list new file mode 100644 index 00000000000..4433e9917e3 --- /dev/null +++ b/utils/check-style/codespell-ignore-words.list @@ -0,0 +1,7 @@ +thenn +fpr +creat +parsering +nd +ect +pullrequest diff --git a/website/templates/index/quickstart.html b/website/templates/index/quickstart.html index add2b66d981..3587113f1fc 100644 --- a/website/templates/index/quickstart.html +++ b/website/templates/index/quickstart.html @@ -21,13 +21,13 @@
- {% highlight "bash" %}{% include "install/deb.sh" %}{% endhighlight %} + {% include "install/deb.sh" %}
- {% highlight "bash" %}{% include "install/rpm.sh" %}{% endhighlight %} + {% include "install/rpm.sh" %}
-
- {% highlight "bash" %}{% include "install/tgz.sh" %}{% endhighlight %} +
+ {% include "install/tgz.sh" %}