diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4acdc293f..09ce72d20ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -409,7 +409,7 @@ ## ClickHouse release 20.6 -### ClickHouse release v20.6.3.28-stable +### ClickHouse release v20.6.3.28-stable #### New Feature @@ -2362,7 +2362,7 @@ No changes compared to v20.4.3.16-stable. * `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) * Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) * Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) +* Now table the first argument of `joinGet` function can be table identifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) * Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) * Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) * `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21cc74bbd2b..783a9f80b66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,25 +59,6 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# cmake 3.9+ needed. -# Usually impractical. -# See also ${ENABLE_THINLTO} -option(ENABLE_IPO "Full link time optimization") - -if(ENABLE_IPO) - cmake_policy(SET CMP0069 NEW) - include(CheckIPOSupported) - check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_NOT_SUPPORTED) - if(IPO_SUPPORTED) - message(STATUS "IPO/LTO is supported, enabling") - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>") - endif() -else() - message(STATUS "IPO/LTO not enabled.") -endif() - # Check that submodules are present only if source was downloaded with git if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") diff --git a/README.md b/README.md index 0c07021630b..03b5c988586 100644 --- a/README.md +++ b/README.md @@ -17,4 +17,6 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events -* [ClickHouse virtual office hours](https://www.eventbrite.com/e/clickhouse-october-virtual-meetup-office-hours-tickets-123129500651) on October 22, 2020. +* [The Second ClickHouse Meetup East (online)](https://www.eventbrite.com/e/the-second-clickhouse-meetup-east-tickets-126787955187) on October 31, 2020. +* [ClickHouse for Enterprise Meetup (online in Russian)](https://arenadata-events.timepad.ru/event/1465249/) on November 10, 2020. + diff --git a/base/common/StringRef.h b/base/common/StringRef.h index 4376876c077..b51b95456cb 100644 --- a/base/common/StringRef.h +++ b/base/common/StringRef.h @@ -51,7 +51,7 @@ struct StringRef }; /// Here constexpr doesn't implicate inline, see https://www.viva64.com/en/w/v1043/ -/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetics +/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetic /// and the UBSan thinks that something like nullptr + 8 is UB. constexpr const inline char empty_string_ref_addr{}; constexpr const inline StringRef EMPTY_STRING_REF{&empty_string_ref_addr, 0}; diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc index 3ac401cb108..6910164d0aa 100644 --- a/cmake/yandex/ya.make.versions.inc +++ b/cmake/yandex/ya.make.versions.inc @@ -11,11 +11,11 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR}) CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH}) CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\") -CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) -CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH}) -# TODO: not supported yet, not sure if ya.make supports arithmetics. +# TODO: not supported yet, not sure if ya.make supports arithmetic. CFLAGS (GLOBAL -DVERSION_INTEGER=0) CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\") diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index ab4857f0387..3aa5cb970db 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -192,7 +192,7 @@ set(SRCS ${HDFS3_SOURCE_DIR}/common/FileWrapper.h ) -# old kernels (< 3.17) doens't have SYS_getrandom. Always use POSIX implementation to have better compatibility +# old kernels (< 3.17) doesn't have SYS_getrandom. Always use POSIX implementation to have better compatibility set_source_files_properties(${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") # target diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index f5638e954a7..1485b0de3ea 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit f5638e954a79f50bac7c7a5deaa5a241e0ce8b5f +Subproject commit 1485b0de3eaa1508dfe49a5ba1e4aa2a71fd8335 diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 4e9259b214a..8fd89d60f85 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -31,10 +31,6 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ && chmod +x dpkg-deb \ && cp dpkg-deb /usr/bin -RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ - && dpkg -i /tmp/arrow-keyring.deb - # Libraries from OS are only needed to test the "unbundled" build (this is not used in production). RUN apt-get update \ && apt-get install \ diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index 604f187cc58..50671011a23 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -1,6 +1,10 @@ # docker build -t yandex/clickhouse-unbundled-builder . FROM yandex/clickhouse-deb-builder +RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ + && dpkg -i /tmp/arrow-keyring.deb + # Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get update \ && apt-get install \ diff --git a/docker/server/.dockerignore b/docker/server/.dockerignore new file mode 100644 index 00000000000..468a8cafb00 --- /dev/null +++ b/docker/server/.dockerignore @@ -0,0 +1,8 @@ +# post / preinstall scripts (not needed, we do it in Dockerfile) +alpine-root/install/* + +# docs (looks useless) +alpine-root/usr/share/doc/* + +# packages, etc. (used by prepare.sh) +alpine-root/tgz-packages/* \ No newline at end of file diff --git a/docker/server/.gitignore b/docker/server/.gitignore new file mode 100644 index 00000000000..4081b5f124c --- /dev/null +++ b/docker/server/.gitignore @@ -0,0 +1 @@ +alpine-root/* \ No newline at end of file diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine new file mode 100644 index 00000000000..fc2756eac8c --- /dev/null +++ b/docker/server/Dockerfile.alpine @@ -0,0 +1,26 @@ +FROM alpine + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 \ + TZ=UTC \ + CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml + +COPY alpine-root/ / + +# from https://github.com/ClickHouse/ClickHouse/blob/master/debian/clickhouse-server.postinst +RUN addgroup clickhouse \ + && adduser -S -H -h /nonexistent -s /bin/false -G clickhouse -g "ClickHouse server" clickhouse \ + && chown clickhouse:clickhouse /var/lib/clickhouse \ + && chmod 700 /var/lib/clickhouse \ + && chown root:clickhouse /var/log/clickhouse-server \ + && chmod 775 /var/log/clickhouse-server \ + && chmod +x /entrypoint.sh \ + && apk add --no-cache su-exec + +EXPOSE 9000 8123 9009 + +VOLUME /var/lib/clickhouse \ + /var/log/clickhouse-server + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh new file mode 100755 index 00000000000..30101225b3e --- /dev/null +++ b/docker/server/alpine-build.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -x + +REPO_CHANNEL="${REPO_CHANNEL:-stable}" # lts / testing / prestable / etc +REPO_URL="${REPO_URL:-"https://repo.yandex.ru/clickhouse/tgz/${REPO_CHANNEL}"}" +VERSION="${VERSION:-20.9.3.45}" + +# where original files live +DOCKER_BUILD_FOLDER="${BASH_SOURCE%/*}" + +# we will create root for our image here +CONTAINER_ROOT_FOLDER="${DOCKER_BUILD_FOLDER}/alpine-root" + +# where to put downloaded tgz +TGZ_PACKAGES_FOLDER="${CONTAINER_ROOT_FOLDER}/tgz-packages" + +# clean up the root from old runs +rm -rf "$CONTAINER_ROOT_FOLDER" + +mkdir -p "$TGZ_PACKAGES_FOLDER" + +PACKAGES=( "clickhouse-client" "clickhouse-server" "clickhouse-common-static" ) + +# download tars from the repo +for package in "${PACKAGES[@]}" +do + wget -q --show-progress "${REPO_URL}/${package}-${VERSION}.tgz" -O "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" +done + +# unpack tars +for package in "${PACKAGES[@]}" +do + tar xvzf "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" --strip-components=2 -C "$CONTAINER_ROOT_FOLDER" +done + +# prepare few more folders +mkdir -p "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/users.d" \ + "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d" \ + "${CONTAINER_ROOT_FOLDER}/var/log/clickhouse-server" \ + "${CONTAINER_ROOT_FOLDER}/var/lib/clickhouse" \ + "${CONTAINER_ROOT_FOLDER}/docker-entrypoint-initdb.d" \ + "${CONTAINER_ROOT_FOLDER}/lib64" + +cp "${DOCKER_BUILD_FOLDER}/docker_related_config.xml" "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d/" +cp "${DOCKER_BUILD_FOLDER}/entrypoint.alpine.sh" "${CONTAINER_ROOT_FOLDER}/entrypoint.sh" + +## get glibc components from ubuntu 20.04 and put them to expected place +docker pull ubuntu:20.04 +ubuntu20image=$(docker create --rm ubuntu:20.04) +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libc.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libdl.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libm.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" + +docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "yandex/clickhouse-server:${VERSION}-alpine" --pull \ No newline at end of file diff --git a/docker/server/entrypoint.alpine.sh b/docker/server/entrypoint.alpine.sh new file mode 100755 index 00000000000..e2edda9ca26 --- /dev/null +++ b/docker/server/entrypoint.alpine.sh @@ -0,0 +1,152 @@ +#!/bin/sh +#set -x + +DO_CHOWN=1 +if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then + DO_CHOWN=0 +fi + +CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" +CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + +# support --user +if [ "$(id -u)" = "0" ]; then + USER=$CLICKHOUSE_UID + GROUP=$CLICKHOUSE_GID + # busybox has setuidgid & chpst buildin + gosu="su-exec $USER:$GROUP" +else + USER="$(id -u)" + GROUP="$(id -g)" + gosu="" + DO_CHOWN=0 +fi + +# set some vars +CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" + +# port is needed to check if clickhouse-server is ready for connections +HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" + +# get CH directories locations +DATA_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=path || true)" +TMP_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=tmp_path || true)" +USER_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=user_files_path || true)" +LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.log || true)" +LOG_DIR="$(dirname $LOG_PATH || true)" +ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.errorlog || true)" +ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" +FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" + +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" +CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" +CLICKHOUSE_DB="${CLICKHOUSE_DB:-}" + +for dir in "$DATA_DIR" \ + "$ERROR_LOG_DIR" \ + "$LOG_DIR" \ + "$TMP_DIR" \ + "$USER_PATH" \ + "$FORMAT_SCHEMA_PATH" +do + # check if variable not empty + [ -z "$dir" ] && continue + # ensure directories exist + if ! mkdir -p "$dir"; then + echo "Couldn't create necessary directory: $dir" + exit 1 + fi + + if [ "$DO_CHOWN" = "1" ]; then + # ensure proper directories permissions + chown -R "$USER:$GROUP" "$dir" + elif [ "$(stat -c %u "$dir")" != "$USER" ]; then + echo "Necessary directory '$dir' isn't owned by user with id '$USER'" + exit 1 + fi +done + +# if clickhouse user is defined - create it (user "default" already exists out of box) +if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ]; then + echo "$0: create new user '$CLICKHOUSE_USER' instead 'default'" + cat < /etc/clickhouse-server/users.d/default-user.xml + + + + + + + + <${CLICKHOUSE_USER}> + default + + ::/0 + + ${CLICKHOUSE_PASSWORD} + default + + + +EOT +fi + +if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then + # Listen only on localhost until the initialization is done + $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- --listen_host=127.0.0.1 & + pid="$!" + + # check if clickhouse is ready to accept connections + # will try to send ping clickhouse via http_port (max 6 retries, with 1 sec timeout and 1 sec delay between retries) + tries=6 + while ! wget --spider -T 1 -q "http://localhost:$HTTP_PORT/ping" 2>/dev/null; do + if [ "$tries" -le "0" ]; then + echo >&2 'ClickHouse init process failed.' + exit 1 + fi + tries=$(( tries-1 )) + sleep 1 + done + + if [ ! -z "$CLICKHOUSE_PASSWORD" ]; then + printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" + fi + + clickhouseclient="clickhouse-client --multiquery -u $CLICKHOUSE_USER $WITH_PASSWORD " + + # create default database, if defined + if [ -n "$CLICKHOUSE_DB" ]; then + echo "$0: create database '$CLICKHOUSE_DB'" + "$clickhouseclient" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB"; + fi + + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; cat "$f" | "$clickhouseclient" ; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "$clickhouseclient"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'Finishing of ClickHouse init process failed.' + exit 1 + fi +fi + +# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments +if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then + exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" +fi + +# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image +exec "$@" diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index bed438a6579..f6c665ff3fd 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -82,6 +82,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV COMMIT_SHA='' ENV PULL_REQUEST_NUMBER='' +ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0 COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 06eb0668fe6..8300c31681e 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -172,6 +172,9 @@ function build ( cd "$FASTTEST_BUILD" time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" +if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then + cp programs/clickhouse "$FASTTEST_OUTPUT/clickhouse" +fi ccache --show-stats ||: ) } @@ -268,7 +271,7 @@ TESTS_TO_SKIP=( 00974_query_profiler # Look at DistributedFilesToInsert, so cannot run in parallel. - 01457_DistributedFilesToInsert + 01460_DistributedFilesToInsert 01541_max_memory_usage_for_user diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 258bc0a95f7..d0cf12baa9e 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -63,7 +63,7 @@ function configure # Make copies of the original db for both servers. Use hardlinks instead # of copying to save space. Before that, remove preprocessed configs and # system tables, because sharing them between servers with hardlinks may - # lead to weird effects. + # lead to weird effects. rm -r left/db ||: rm -r right/db ||: rm -r db0/preprocessed_configs ||: @@ -77,15 +77,12 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed - # Disable percpu arenas because they segfault when the process is bound to - # a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939 - # - # About the jemalloc settings: + # Change the jemalloc settings here. # https://github.com/jemalloc/jemalloc/wiki/Getting-Started - export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" + export MALLOC_CONF="confirm_conf:true" set -m # Spawn servers in their own process groups - + left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -211,7 +208,7 @@ function run_tests echo test "$test_name" # Don't profile if we're past the time limit. - # Use awk because bash doesn't support floating point arithmetics. + # Use awk because bash doesn't support floating point arithmetic. profile_seconds=$(awk "BEGIN { print ($profile_seconds_left > 0 ? 10 : 0) }") TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n") @@ -544,10 +541,10 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') as select abs(diff) > report_threshold and abs(diff) > stat_threshold as changed_fail, abs(diff) > report_threshold - 0.05 and abs(diff) > stat_threshold as changed_show, - + not changed_fail and stat_threshold > report_threshold + 0.10 as unstable_fail, not changed_show and stat_threshold > report_threshold - 0.05 as unstable_show, - + left, right, diff, stat_threshold, if(report_threshold > 0, report_threshold, 0.10) as report_threshold, query_metric_stats.test test, query_metric_stats.query_index query_index, @@ -770,7 +767,7 @@ create table all_tests_report engine File(TSV, 'report/all-queries.tsv') as -- The threshold for 2) is significantly larger than the threshold for 1), to -- avoid jitter. create view shortness - as select + as select (test, query_index) in (select * from file('analyze/marked-short-queries.tsv', TSV, 'test text, query_index int')) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index e69fdc0fce0..8793f3d22b4 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -17,14 +17,24 @@ service clickhouse-server start && sleep 5 if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then SKIP_LIST_OPT="--use-skip-list" fi -# We can have several additional options so we path them as array because it's -# more idiologically correct. -read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" function run_tests() { + # We can have several additional options so we path them as array because it's + # more idiologically correct. + read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" + + # Skip these tests, because they fail when we rerun them multiple times + if [ "$NUM_TRIES" -gt "1" ]; then + ADDITIONAL_OPTIONS+=('--skip') + ADDITIONAL_OPTIONS+=('00000_no_tests_to_skip') + fi + for i in $(seq 1 $NUM_TRIES); do - clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt + clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt + if [ ${PIPESTATUS[0]} -ne "0" ]; then + break; + fi done } diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 9565e39598c..06241d6d497 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.57 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.59 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 07d843f724a..06216517db8 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -18,4 +18,14 @@ toc_title: Cloud - Encryption and isolation - Automated maintenance +## Altinity.Cloud {#altinity.cloud} + +[Altinity.Cloud](https://altinity.com/cloud-database/) is a fully managed ClickHouse-as-a-Service for the Amazon public cloud. +- Fast deployment of ClickHouse clusters on Amazon resources +- Easy scale-out/scale-in as well as vertical scaling of nodes +- Isolated per-tenant VPCs with public endpoint or VPC peering +- Configurable storage types and volume configurations +- Cross-AZ scaling for performance and high availability +- Built-in monitoring and SQL query editor + {## [Original article](https://clickhouse.tech/docs/en/commercial/cloud/) ##} diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md index dd14ee3b4b1..b0901ee6f6e 100644 --- a/docs/en/engines/table-engines/integrations/rabbitmq.md +++ b/docs/en/engines/table-engines/integrations/rabbitmq.md @@ -51,7 +51,7 @@ Optional parameters: - `rabbitmq_row_delimiter` – Delimiter character, which ends the message. - `rabbitmq_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. - `rabbitmq_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. -- `rabbitmq_num_queues` – The number of queues per consumer. Default: `1`. Specify more queues if the capacity of one queue per consumer is insufficient. +- `rabbitmq_num_queues` – Total number of queues. Default: `1`. Increasing this number can significantly improve performance. - `rabbitmq_queue_base` - Specify a hint for queue names. Use cases of this setting are described below. - `rabbitmq_deadletter_exchange` - Specify name for a [dead letter exchange](https://www.rabbitmq.com/dlx.html). You can create another table with this exchange name and collect messages in cases when they are republished to dead letter exchange. By default dead letter exchange is not specified. - `rabbitmq_persistent` - If set to 1 (true), in insert query delivery mode will be set to 2 (marks messages as 'persistent'). Default: `0`. @@ -148,4 +148,5 @@ Example: - `_channel_id` - ChannelID, on which consumer, who received the message, was declared. - `_delivery_tag` - DeliveryTag of the received message. Scoped per channel. - `_redelivered` - `redelivered` flag of the message. -- `_message_id` - MessageID of the received message; non-empty if was set, when message was published. +- `_message_id` - messageID of the received message; non-empty if was set, when message was published. +- `_timestamp` - timestamp of the received message; non-empty if was set, when message was published. diff --git a/docs/en/faq/integration/json-import.md b/docs/en/faq/integration/json-import.md index 067b407a079..fb94f226f2b 100644 --- a/docs/en/faq/integration/json-import.md +++ b/docs/en/faq/integration/json-import.md @@ -30,4 +30,4 @@ Instead of inserting data manually, you might consider to use one of [client lib - `input_format_import_nested_json` allows to insert nested JSON objects into columns of [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) type. !!! note "Note" - Settings are specified as `GET` parameters for the HTTP interface or as additional command-line arguments prefixed with `--` for the CLI interface. + Settings are specified as `GET` parameters for the HTTP interface or as additional command-line arguments prefixed with `--` for the `CLI` interface. \ No newline at end of file diff --git a/docs/en/getting-started/example-datasets/amplab-benchmark.md b/docs/en/getting-started/example-datasets/amplab-benchmark.md index 1b740f6194c..27ddfd27f78 100644 --- a/docs/en/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/en/getting-started/example-datasets/amplab-benchmark.md @@ -1,5 +1,5 @@ --- -toc_priority: 17 +toc_priority: 19 toc_title: AMPLab Big Data Benchmark --- diff --git a/docs/en/getting-started/example-datasets/criteo.md b/docs/en/getting-started/example-datasets/criteo.md index a38a1b14210..261d1606fa4 100644 --- a/docs/en/getting-started/example-datasets/criteo.md +++ b/docs/en/getting-started/example-datasets/criteo.md @@ -1,5 +1,5 @@ --- -toc_priority: 19 +toc_priority: 18 toc_title: Terabyte Click Logs from Criteo --- diff --git a/docs/en/getting-started/example-datasets/index.md b/docs/en/getting-started/example-datasets/index.md index eedf4196b23..35ac90f9beb 100644 --- a/docs/en/getting-started/example-datasets/index.md +++ b/docs/en/getting-started/example-datasets/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: Example Datasets -toc_priority: 15 +toc_priority: 14 toc_title: Introduction --- @@ -18,4 +18,4 @@ The list of documented datasets: - [New York Taxi Data](../../getting-started/example-datasets/nyc-taxi.md) - [OnTime](../../getting-started/example-datasets/ontime.md) -[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets) +[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets) \ No newline at end of file diff --git a/docs/en/getting-started/example-datasets/metrica.md b/docs/en/getting-started/example-datasets/metrica.md index 4131dca78fe..b036973b255 100644 --- a/docs/en/getting-started/example-datasets/metrica.md +++ b/docs/en/getting-started/example-datasets/metrica.md @@ -1,5 +1,5 @@ --- -toc_priority: 14 +toc_priority: 15 toc_title: Yandex.Metrica Data --- diff --git a/docs/en/getting-started/example-datasets/nyc-taxi.md b/docs/en/getting-started/example-datasets/nyc-taxi.md index a180f19be8a..9b9a12ba724 100644 --- a/docs/en/getting-started/example-datasets/nyc-taxi.md +++ b/docs/en/getting-started/example-datasets/nyc-taxi.md @@ -1,5 +1,5 @@ --- -toc_priority: 16 +toc_priority: 20 toc_title: New York Taxi Data --- diff --git a/docs/en/getting-started/example-datasets/ontime.md b/docs/en/getting-started/example-datasets/ontime.md index 91b6913e223..c2c8d5d930e 100644 --- a/docs/en/getting-started/example-datasets/ontime.md +++ b/docs/en/getting-started/example-datasets/ontime.md @@ -1,5 +1,5 @@ --- -toc_priority: 15 +toc_priority: 21 toc_title: OnTime --- diff --git a/docs/en/getting-started/example-datasets/star-schema.md b/docs/en/getting-started/example-datasets/star-schema.md index 7940807ee49..8a019e7ded3 100644 --- a/docs/en/getting-started/example-datasets/star-schema.md +++ b/docs/en/getting-started/example-datasets/star-schema.md @@ -1,5 +1,5 @@ --- -toc_priority: 20 +toc_priority: 16 toc_title: Star Schema Benchmark --- diff --git a/docs/en/getting-started/example-datasets/wikistat.md b/docs/en/getting-started/example-datasets/wikistat.md index 874af8c9423..619711582f4 100644 --- a/docs/en/getting-started/example-datasets/wikistat.md +++ b/docs/en/getting-started/example-datasets/wikistat.md @@ -1,5 +1,5 @@ --- -toc_priority: 18 +toc_priority: 17 toc_title: WikiStat --- diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index d96d48bdca3..d310705d1c1 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -460,7 +460,7 @@ See also the [JSONEachRow](#jsoneachrow) format. ## JSONString {#jsonstring} -Differs from JSON only in that data fields are output in strings, not in typed json values. +Differs from JSON only in that data fields are output in strings, not in typed JSON values. Example: @@ -596,7 +596,7 @@ When inserting the data, you should provide a separate JSON value for each row. ## JSONEachRowWithProgress {#jsoneachrowwithprogress} ## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} -Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield progress information as JSON objects. +Differs from `JSONEachRow`/`JSONStringEachRow` in that ClickHouse will also yield progress information as JSON values. ```json {"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} @@ -608,7 +608,7 @@ Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield pr ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} ## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} -Differs from JSONCompactEachRow/JSONCompactStringEachRow in that the column names and types are written as the first two rows. +Differs from `JSONCompactEachRow`/`JSONCompactStringEachRow` in that the column names and types are written as the first two rows. ```json ["'hello'", "multiply(42, number)", "range(5)"] diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index bb98c5f936b..4e8347c9a6e 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -6,7 +6,7 @@ toc_title: Client Libraries # Client Libraries from Third-party Developers {#client-libraries-from-third-party-developers} !!! warning "Disclaimer" - Yandex does **not** maintain the libraries listed below and haven’t done any extensive testing to ensure their quality. + Yandex does **not** maintain the libraries listed below and hasn’t done any extensive testing to ensure their quality. - Python - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md new file mode 100644 index 00000000000..45533d3733f --- /dev/null +++ b/docs/en/operations/opentelemetry.md @@ -0,0 +1,69 @@ +--- +toc_priority: 62 +toc_title: OpenTelemetry Support +--- + +# [experimental] OpenTelemetry Support + +[OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting +traces and metrics from distributed application. ClickHouse has some support +for OpenTelemetry. + +!!! warning "Warning" +This is an experimental feature that will change in backwards-incompatible ways in the future releases. + + +## Supplying Trace Context to ClickHouse + +ClickHouse accepts trace context HTTP headers, as described by +the [W3C recommendation](https://www.w3.org/TR/trace-context/). +It also accepts trace context over native protocol that is used for +communication between ClickHouse servers or between the client and server. +For manual testing, trace context headers conforming to the Trace Context +recommendation can be supplied to `clickhouse-client` using +`--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags. + +If no parent trace context is supplied, ClickHouse can start a new trace, with +probability controlled by the `opentelemetry_start_trace_probability` setting. + + +## Propagating the Trace Context + +The trace context is propagated to downstream services in the following cases: + +* Queries to remote ClickHouse servers, such as when using `Distributed` table + engine. + +* `URL` table function. Trace context information is sent in HTTP headers. + + +## Tracing the ClickHouse Itself + +ClickHouse creates _trace spans_ for each query and some of the query execution +stages, such as query planning or distributed queries. + +To be useful, the tracing information has to be exported to a monitoring system +that supports OpenTelemetry, such as Jaeger or Prometheus. ClickHouse avoids +a dependency on a particular monitoring system, instead only +providing the tracing data conforming to the standard. A natural way to do so +in an SQL RDBMS is a system table. OpenTelemetry trace span information +[required by the standard](https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/overview.md#span) +is stored in the system table called `system.opentelemetry_span_log`. + +The table must be enabled in the server configuration, see the `opentelemetry_span_log` +element in the default config file `config.xml`. It is enabled by default. + +The table has the following columns: + +- `trace_id` +- `span_id` +- `parent_span_id` +- `operation_name` +- `start_time` +- `finish_time` +- `finish_date` +- `attribute.name` +- `attribute.values` + +The tags or attributes are saved as two parallel arrays, containing the keys +and values. Use `ARRAY JOIN` to work with them. diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index ddd5d327d59..32b2bdf2133 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -20,8 +20,8 @@ The `system.query_log` table registers two kinds of queries: Each query creates one or two rows in the `query_log` table, depending on the status (see the `type` column) of the query: -1. If the query execution was successful, two rows with the `QueryStart` and `QueryFinish` types are created . -2. If an error occurred during query processing, two events with the `QueryStart` and `ExceptionWhileProcessing` types are created . +1. If the query execution was successful, two rows with the `QueryStart` and `QueryFinish` types are created. +2. If an error occurred during query processing, two events with the `QueryStart` and `ExceptionWhileProcessing` types are created. 3. If an error occurred before launching the query, a single event with the `ExceptionBeforeStart` type is created. Columns: @@ -37,8 +37,8 @@ Columns: - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Start time of query execution. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Start time of query execution with microsecond precision. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution in milliseconds. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it’s `read_rows` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn’t affect this value. -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it’s `read_bytes` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn’t affect this value. +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it’s `read_rows` value, and the server-initiator of the query summarizes all received and local values. The cache volumes don’t affect this value. +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it’s `read_bytes` value, and the server-initiator of the query summarizes all received and local values. The cache volumes don’t affect this value. - `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. - `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of rows in a result of the `SELECT` query, or a number of rows in the `INSERT` query. diff --git a/docs/en/operations/system-tables/query_thread_log.md b/docs/en/operations/system-tables/query_thread_log.md index a6f090e99f3..0ae2e7d5d3b 100644 --- a/docs/en/operations/system-tables/query_thread_log.md +++ b/docs/en/operations/system-tables/query_thread_log.md @@ -1,6 +1,6 @@ # system.query_thread_log {#system_tables-query_thread_log} -Contains information about threads which execute queries, for example, thread name, thread start time, duration of query processing. +Contains information about threads that execute queries, for example, thread name, thread start time, duration of query processing. To start logging: diff --git a/docs/en/operations/system-tables/text_log.md b/docs/en/operations/system-tables/text_log.md index 34f37c2cae7..f5f53c95653 100644 --- a/docs/en/operations/system-tables/text_log.md +++ b/docs/en/operations/system-tables/text_log.md @@ -1,6 +1,6 @@ # system.text_log {#system_tables-text_log} -Contains logging entries. Logging level which goes to this table can be limited with `text_log.level` server setting. +Contains logging entries. The logging level which goes to this table can be limited to the `text_log.level` server setting. Columns: diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md index bc955ebe4de..8107f60b808 100644 --- a/docs/en/operations/system-tables/trace_log.md +++ b/docs/en/operations/system-tables/trace_log.md @@ -18,7 +18,7 @@ Columns: - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. - When connecting to server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. + When connecting to the server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. - `timer_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Timer type: diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index 8a4b9ae7886..7c0c5b00c0d 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -80,4 +80,4 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## See Also {#see-also} - [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator -- [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 44008fbbcbe..bc3f5ca4345 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -6,7 +6,7 @@ toc_title: Encoding # Encoding Functions {#encoding-functions} ## char {#char} - + Returns the string with the length as the number of passed arguments and each byte has the value of corresponding argument. Accepts multiple arguments of numeric types. If the value of argument is out of range of UInt8 data type, it is converted to UInt8 with possible rounding and overflow. **Syntax** diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 1145efbbc5f..2cc80dcffc1 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -551,7 +551,7 @@ formatReadableTimeDelta(column[, maximum_unit]) **Parameters** - `column` — A column with numeric time delta. -- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. +- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. Example: @@ -626,7 +626,12 @@ neighbor(column, offset[, default_value]) ``` The result of the function depends on the affected data blocks and the order of data in the block. -If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +!!! warning "Warning" + It can reach the neighbor rows only inside the currently processed data block. + +The rows order used during the calculation of `neighbor` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with ORDER BY and call the function from outside the subquery. **Parameters** @@ -731,8 +736,13 @@ Result: Calculates the difference between successive row values ​​in the data block. Returns 0 for the first row and the difference from the previous row for each subsequent row. +!!! warning "Warning" + It can reach the previos row only inside the currently processed data block. + The result of the function depends on the affected data blocks and the order of data in the block. -If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +The rows order used during the calculation of `runningDifference` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with ORDER BY and call the function from outside the subquery. Example: @@ -1584,7 +1594,7 @@ isDecimalOverflow(d, [p]) **Parameters** - `d` — value. [Decimal](../../sql-reference/data-types/decimal.md). -- `p` — precision. Optional. If omitted, the initial presicion of the first argument is used. Using of this paratemer could be helpful for data extraction to another DBMS or file. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `p` — precision. Optional. If omitted, the initial precision of the first argument is used. Using of this paratemer could be helpful for data extraction to another DBMS or file. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). **Returned values** diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md index f608c643ee8..01a61c65b67 100644 --- a/docs/en/sql-reference/functions/uuid-functions.md +++ b/docs/en/sql-reference/functions/uuid-functions.md @@ -61,6 +61,54 @@ SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS uuid └──────────────────────────────────────┘ ``` +## toUUIDOrNull (x) {#touuidornull-x} + +It takes an argument of type String and tries to parse it into UUID. If failed, returns NULL. + +``` sql +toUUIDOrNull(String) +``` + +**Returned value** + +The Nullable(UUID) type value. + +**Usage example** + +``` sql +SELECT toUUIDOrNull('61f0c404-5cb3-11e7-907b-a6006ad3dba0T') AS uuid +``` + +``` text +┌─uuid─┐ +│ ᴺᵁᴸᴸ │ +└──────┘ +``` + +## toUUIDOrZero (x) {#touuidorzero-x} + +It takes an argument of type String and tries to parse it into UUID. If failed, returns zero UUID. + +``` sql +toUUIDOrZero(String) +``` + +**Returned value** + +The UUID type value. + +**Usage example** + +``` sql +SELECT toUUIDOrZero('61f0c404-5cb3-11e7-907b-a6006ad3dba0T') AS uuid +``` + +``` text +┌─────────────────────────────────uuid─┐ +│ 00000000-0000-0000-0000-000000000000 │ +└──────────────────────────────────────┘ +``` + ## UUIDStringToNum {#uuidstringtonum} Accepts a string containing 36 characters in the format `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`, and returns it as a set of bytes in a [FixedString(16)](../../sql-reference/data-types/fixedstring.md). diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 3fe3384fffc..262ae00dc95 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 37 +toc_priority: 38 toc_title: Operators --- @@ -169,7 +169,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **See Also** - [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type -- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions ## Logical Negation Operator {#logical-negation-operator} diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index 60b7375f76d..30603122096 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 36 +toc_priority: 35 toc_title: ALTER --- diff --git a/docs/en/sql-reference/statements/alter/sample-by.md b/docs/en/sql-reference/statements/alter/sample-by.md index bea6364e4ea..df8ff90f196 100644 --- a/docs/en/sql-reference/statements/alter/sample-by.md +++ b/docs/en/sql-reference/statements/alter/sample-by.md @@ -5,16 +5,16 @@ toc_title: SAMPLE BY # Manipulating Sampling-Key Expressions {#manipulations-with-sampling-key-expressions} +Syntax: + ``` sql ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY SAMPLE BY new_expression ``` The command changes the [sampling key](../../../engines/table-engines/mergetree-family/mergetree.md) of the table to `new_expression` (an expression or a tuple of expressions). -The command is lightweight in a sense that it only changes metadata. The primary key must contain the new sample key. +The command is lightweight in the sense that it only changes metadata. The primary key must contain the new sample key. !!! note "Note" - It only works for tables in the [`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) family (including -[replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). - - + It only works for tables in the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) family (including +[replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). \ No newline at end of file diff --git a/docs/en/sql-reference/statements/attach.md b/docs/en/sql-reference/statements/attach.md index 6260e15cfa9..035441ef5f1 100644 --- a/docs/en/sql-reference/statements/attach.md +++ b/docs/en/sql-reference/statements/attach.md @@ -1,5 +1,5 @@ --- -toc_priority: 42 +toc_priority: 40 toc_title: ATTACH --- diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md index b55955eeb40..450447acaf8 100644 --- a/docs/en/sql-reference/statements/check-table.md +++ b/docs/en/sql-reference/statements/check-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 43 +toc_priority: 41 toc_title: CHECK --- diff --git a/docs/en/sql-reference/statements/create/database.md b/docs/en/sql-reference/statements/create/database.md index e874672471d..bdb31d44b0b 100644 --- a/docs/en/sql-reference/statements/create/database.md +++ b/docs/en/sql-reference/statements/create/database.md @@ -1,5 +1,5 @@ --- -toc_priority: 1 +toc_priority: 35 toc_title: DATABASE --- diff --git a/docs/en/sql-reference/statements/create/dictionary.md b/docs/en/sql-reference/statements/create/dictionary.md index 5313c59396e..b1098c54703 100644 --- a/docs/en/sql-reference/statements/create/dictionary.md +++ b/docs/en/sql-reference/statements/create/dictionary.md @@ -1,5 +1,5 @@ --- -toc_priority: 4 +toc_priority: 38 toc_title: DICTIONARY --- diff --git a/docs/en/sql-reference/statements/create/index.md b/docs/en/sql-reference/statements/create/index.md index 71ace2b664c..902a4348bac 100644 --- a/docs/en/sql-reference/statements/create/index.md +++ b/docs/en/sql-reference/statements/create/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: CREATE -toc_priority: 35 +toc_priority: 34 toc_title: Overview --- diff --git a/docs/en/sql-reference/statements/create/quota.md b/docs/en/sql-reference/statements/create/quota.md index 7919dc0f6db..29752050b69 100644 --- a/docs/en/sql-reference/statements/create/quota.md +++ b/docs/en/sql-reference/statements/create/quota.md @@ -1,5 +1,5 @@ --- -toc_priority: 8 +toc_priority: 42 toc_title: QUOTA --- diff --git a/docs/en/sql-reference/statements/create/role.md b/docs/en/sql-reference/statements/create/role.md index 8a682875d63..19db2b79b61 100644 --- a/docs/en/sql-reference/statements/create/role.md +++ b/docs/en/sql-reference/statements/create/role.md @@ -1,5 +1,5 @@ --- -toc_priority: 6 +toc_priority: 40 toc_title: ROLE --- diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md index b1e7c8e9006..9f8b6b87d3b 100644 --- a/docs/en/sql-reference/statements/create/row-policy.md +++ b/docs/en/sql-reference/statements/create/row-policy.md @@ -1,5 +1,5 @@ --- -toc_priority: 7 +toc_priority: 41 toc_title: ROW POLICY --- diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md index 6fcd1d4e840..196c336bb4d 100644 --- a/docs/en/sql-reference/statements/create/settings-profile.md +++ b/docs/en/sql-reference/statements/create/settings-profile.md @@ -1,5 +1,5 @@ --- -toc_priority: 9 +toc_priority: 43 toc_title: SETTINGS PROFILE --- diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 9aecc6c07f7..82326bf51cf 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -1,5 +1,5 @@ --- -toc_priority: 2 +toc_priority: 36 toc_title: TABLE --- @@ -121,7 +121,7 @@ Defines storage time for values. Can be specified only for MergeTree-family tabl ## Column Compression Codecs {#codecs} -By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. +By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. You can also define the compression method for each individual column in the `CREATE TABLE` query. @@ -138,7 +138,7 @@ ENGINE = ... ``` -The `Default` codec can be specified to reference default compression which may dependend on different settings (and properties of data) in runtime. +The `Default` codec can be specified to reference default compression which may depend on different settings (and properties of data) in runtime. Example: `value UInt64 CODEC(Default)` — the same as lack of codec specification. Also you can remove current CODEC from the column and use default compression from config.xml: @@ -149,7 +149,7 @@ ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. -To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. +To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. !!! warning "Warning" You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 98c29363102..273882fd639 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -1,5 +1,5 @@ --- -toc_priority: 5 +toc_priority: 39 toc_title: USER --- diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index dc1200e5979..17a6c26c084 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -1,5 +1,5 @@ --- -toc_priority: 3 +toc_priority: 37 toc_title: VIEW --- diff --git a/docs/en/sql-reference/statements/describe-table.md b/docs/en/sql-reference/statements/describe-table.md index b7ffec1db07..bc197bf0f72 100644 --- a/docs/en/sql-reference/statements/describe-table.md +++ b/docs/en/sql-reference/statements/describe-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 44 +toc_priority: 42 toc_title: DESCRIBE --- diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 0d236c132df..62a7c0cc1e0 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -1,5 +1,5 @@ --- -toc_priority: 45 +toc_priority: 43 toc_title: DETACH --- diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index de4eda64ef6..4317a20419e 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -1,88 +1,100 @@ --- -toc_priority: 46 +toc_priority: 44 toc_title: DROP --- # DROP Statements {#drop} -Deletes existing entity. If `IF EXISTS` clause is specified, these queries doesn’t return an error if the entity doesn’t exist. +Deletes existing entity. If the `IF EXISTS` clause is specified, these queries don’t return an error if the entity doesn’t exist. ## DROP DATABASE {#drop-database} +Deletes all tables inside the `db` database, then deletes the `db` database itself. + +Syntax: + ``` sql DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Deletes all tables inside the `db` database, then deletes the ‘db’ database itself. - ## DROP TABLE {#drop-table} +Deletes the table. + +Syntax: + ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Deletes the table. - ## DROP DICTIONARY {#drop-dictionary} +Deletes the dictionary. + +Syntax: + ``` sql DROP DICTIONARY [IF EXISTS] [db.]name ``` -Deletes the dictionary. - ## DROP USER {#drop-user-statement} +Deletes a user. + +Syntax: + ``` sql DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a user. - ## DROP ROLE {#drop-role-statement} +Deletes a role. The deleted role is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a role. - -Deleted role is revoked from all the entities where it was assigned. - ## DROP ROW POLICY {#drop-row-policy-statement} +Deletes a row policy. Deleted row policy is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] ``` -Deletes a row policy. - -Deleted row policy is revoked from all the entities where it was assigned. - ## DROP QUOTA {#drop-quota-statement} +Deletes a quota. The deleted quota is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a quota. - -Deleted quota is revoked from all the entities where it was assigned. - ## DROP SETTINGS PROFILE {#drop-settings-profile-statement} +Deletes a settings profile. The deleted settings profile is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a settings profile. - -Deleted settings profile is revoked from all the entities where it was assigned. - ## DROP VIEW {#drop-view} +Deletes a view. Views can be deleted by a `DROP TABLE` command as well but `DROP VIEW` checks that `[db.]name` is a view. + +Syntax: + ``` sql DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Deletes a view. Views can be deleted by a `DROP TABLE` command as well but `DROP VIEW` checks that `[db.]name` is a view. +[Оriginal article](https://clickhouse.tech/docs/en/sql-reference/statements/drop/) \ No newline at end of file diff --git a/docs/en/sql-reference/statements/exists.md b/docs/en/sql-reference/statements/exists.md index 2e8072125c7..3b0f4b66343 100644 --- a/docs/en/sql-reference/statements/exists.md +++ b/docs/en/sql-reference/statements/exists.md @@ -1,5 +1,5 @@ --- -toc_priority: 47 +toc_priority: 45 toc_title: EXISTS --- diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index c3e18ca3c75..f3829de2fbb 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -1,5 +1,5 @@ --- -toc_priority: 39 +toc_priority: 38 toc_title: GRANT --- diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index a2afbb2e10a..b49314a1785 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -1,5 +1,5 @@ --- -toc_priority: 34 +toc_priority: 33 toc_title: INSERT INTO --- diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 1e2d2e925f7..d3f2d9bb5c6 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -1,5 +1,5 @@ --- -toc_priority: 48 +toc_priority: 46 toc_title: KILL --- diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 374ef926d6c..a67f282e793 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -1,5 +1,5 @@ --- -toc_priority: 49 +toc_priority: 47 toc_title: OPTIMIZE --- diff --git a/docs/en/sql-reference/statements/rename.md b/docs/en/sql-reference/statements/rename.md index 315aaac40b5..4f14ad016a3 100644 --- a/docs/en/sql-reference/statements/rename.md +++ b/docs/en/sql-reference/statements/rename.md @@ -1,5 +1,5 @@ --- -toc_priority: 50 +toc_priority: 48 toc_title: RENAME --- diff --git a/docs/en/sql-reference/statements/revoke.md b/docs/en/sql-reference/statements/revoke.md index 8269c1f34a9..71a76546096 100644 --- a/docs/en/sql-reference/statements/revoke.md +++ b/docs/en/sql-reference/statements/revoke.md @@ -1,5 +1,5 @@ --- -toc_priority: 40 +toc_priority: 39 toc_title: REVOKE --- diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 07be8c2bf45..3107f791eb9 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -1,7 +1,7 @@ --- title: SELECT Query toc_folder_title: SELECT -toc_priority: 33 +toc_priority: 32 toc_title: Overview --- diff --git a/docs/en/sql-reference/statements/set-role.md b/docs/en/sql-reference/statements/set-role.md index 564c8ec859f..cf14a9c6d75 100644 --- a/docs/en/sql-reference/statements/set-role.md +++ b/docs/en/sql-reference/statements/set-role.md @@ -1,5 +1,5 @@ --- -toc_priority: 52 +toc_priority: 51 toc_title: SET ROLE --- diff --git a/docs/en/sql-reference/statements/set.md b/docs/en/sql-reference/statements/set.md index 4a5bbf0baf6..c6c8d28257d 100644 --- a/docs/en/sql-reference/statements/set.md +++ b/docs/en/sql-reference/statements/set.md @@ -1,5 +1,5 @@ --- -toc_priority: 51 +toc_priority: 49 toc_title: SET --- diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 81aca1261de..e1f2ef3488a 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -1,5 +1,5 @@ --- -toc_priority: 38 +toc_priority: 37 toc_title: SHOW --- diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 8230177f249..75303fde19e 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -1,5 +1,5 @@ --- -toc_priority: 37 +toc_priority: 36 toc_title: SYSTEM --- diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index a8cf802616f..a13936c88ab 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -1,5 +1,5 @@ --- -toc_priority: 53 +toc_priority: 52 toc_title: TRUNCATE --- diff --git a/docs/en/sql-reference/statements/use.md b/docs/en/sql-reference/statements/use.md index 2932542f052..841c23d333d 100644 --- a/docs/en/sql-reference/statements/use.md +++ b/docs/en/sql-reference/statements/use.md @@ -1,5 +1,5 @@ --- -toc_priority: 54 +toc_priority: 53 toc_title: USE --- diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 6cbc0e6743b..e6b0309c456 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -1,3 +1,8 @@ +--- +toc_priority: 1 +toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u043e\u0431\u043b\u0430\u0447\u043d\u044b\u0445\u0020\u0443\u0441\u043b\u0443\u0433\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Поставщики облачных услуг ClickHouse {#clickhouse-cloud-service-providers} !!! info "Инфо" diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md index e66b20f764a..0d8ddb6f795 100644 --- a/docs/ru/development/architecture.md +++ b/docs/ru/development/architecture.md @@ -1,3 +1,8 @@ +--- +toc_priority: 62 +toc_title: "\u041e\u0431\u0437\u043e\u0440\u0020\u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Обзор архитектуры ClickHouse {#overview-of-clickhouse-architecture} ClickHouse - полноценная колоночная СУБД. Данные хранятся в колонках, а в процессе обработки - в массивах (векторах или фрагментах (chunk’ах) колонок). По возможности операции выполняются на массивах, а не на индивидуальных значениях. Это называется “векторизованное выполнения запросов” (vectorized query execution), и помогает снизить стоимость фактической обработки данных. diff --git a/docs/ru/development/browse-code.md b/docs/ru/development/browse-code.md index 814b213a6a7..ac17cf0e6f5 100644 --- a/docs/ru/development/browse-code.md +++ b/docs/ru/development/browse-code.md @@ -1,3 +1,9 @@ +--- +toc_priority: 71 +toc_title: "\u041d\u0430\u0432\u0438\u0433\u0430\u0446\u0438\u044f\u0020\u043f\u043e\u0020\u043a\u043e\u0434\u0443\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + + # Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse} Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md index c640ecee79d..e65ab4819e8 100644 --- a/docs/ru/development/contrib.md +++ b/docs/ru/development/contrib.md @@ -1,3 +1,9 @@ +--- +toc_priority: 70 +toc_title: "\u0418\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c\u044b\u0435\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0435\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438" +--- + + # Используемые сторонние библиотеки {#ispolzuemye-storonnie-biblioteki} | Библиотека | Лицензия | diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 9edcd4cf9f9..b0c84d9de7c 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: "\u0418\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u044f\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Инструкция для разработчиков Сборка ClickHouse поддерживается на Linux, FreeBSD, Mac OS X. diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index 951537e8113..671293a7bbd 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -1,3 +1,9 @@ +--- +toc_priority: 68 +toc_title: "\u041a\u0430\u043a\u0020\u043f\u0438\u0441\u0430\u0442\u044c\u0020\u043a\u043e\u0434\u0020\u043d\u0430\u0020\u0043\u002b\u002b" +--- + + # Как писать код на C++ {#kak-pisat-kod-na-c} ## Общее {#obshchee} diff --git a/docs/ru/engines/database-engines/index.md b/docs/ru/engines/database-engines/index.md index 79be85518e2..d3dd729e302 100644 --- a/docs/ru/engines/database-engines/index.md +++ b/docs/ru/engines/database-engines/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0431\u0430\u0437\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_priority: 27 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + + # Движки баз данных {#dvizhki-baz-dannykh} Движки баз данных обеспечивают работу с таблицами. diff --git a/docs/ru/engines/database-engines/lazy.md b/docs/ru/engines/database-engines/lazy.md index c1538ec91bc..c01aae0284e 100644 --- a/docs/ru/engines/database-engines/lazy.md +++ b/docs/ru/engines/database-engines/lazy.md @@ -1,3 +1,8 @@ +--- +toc_priority: 31 +toc_title: Lazy +--- + # Lazy {#lazy} Сохраняет таблицы только в оперативной памяти `expiration_time_in_seconds` через несколько секунд после последнего доступа. Может использоваться только с таблицами \*Log. diff --git a/docs/ru/engines/database-engines/mysql.md b/docs/ru/engines/database-engines/mysql.md index e293cb6ec77..2f8335de73a 100644 --- a/docs/ru/engines/database-engines/mysql.md +++ b/docs/ru/engines/database-engines/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 30 +toc_title: MySQL +--- + # MySQL {#mysql} Позволяет подключаться к базам данных на удалённом MySQL сервере и выполнять запросы `INSERT` и `SELECT` для обмена данными между ClickHouse и MySQL. diff --git a/docs/ru/engines/index.md b/docs/ru/engines/index.md index 333e65dcb7b..28ccc8bcfe6 100644 --- a/docs/ru/engines/index.md +++ b/docs/ru/engines/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: "\u0414\u0432\u0438\u0436\u043A\u0438" +toc_folder_title: "\u0045\u006e\u0067\u0069\u006e\u0065\u0073" +toc_hidden: true toc_priority: 25 +toc_title: hidden --- - - diff --git a/docs/ru/engines/table-engines/index.md b/docs/ru/engines/table-engines/index.md index 6a954313c60..740588c50a4 100644 --- a/docs/ru/engines/table-engines/index.md +++ b/docs/ru/engines/table-engines/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446" +toc_priority: 26 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + + # Движки таблиц {#table_engines} Движок таблицы (тип таблицы) определяет: diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index b5234b38d7d..bd8e760fce4 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -1,3 +1,8 @@ +--- +toc_priority: 4 +toc_title: HDFS +--- + # HDFS {#table_engines-hdfs} Управляет данными в HDFS. Данный движок похож на движки [File](../special/file.md#table_engines-file) и [URL](../special/url.md#table_engines-url). diff --git a/docs/ru/engines/table-engines/integrations/index.md b/docs/ru/engines/table-engines/integrations/index.md index 0186f6cf19e..02189cf9e55 100644 --- a/docs/ru/engines/table-engines/integrations/index.md +++ b/docs/ru/engines/table-engines/integrations/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: Integrations +toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446\u0020\u0434\u043b\u044f\u0020\u0438\u043d\u0442\u0435\u0433\u0440\u0430\u0446\u0438\u0438" toc_priority: 30 --- diff --git a/docs/ru/engines/table-engines/integrations/jdbc.md b/docs/ru/engines/table-engines/integrations/jdbc.md index cded0c51fa0..d7d438e0633 100644 --- a/docs/ru/engines/table-engines/integrations/jdbc.md +++ b/docs/ru/engines/table-engines/integrations/jdbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 2 +toc_title: JDBC +--- + # JDBC {#table-engine-jdbc} Позволяет ClickHouse подключаться к внешним базам данных с помощью [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity). diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index 8c47e97f497..bcca349f743 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -1,3 +1,8 @@ +--- +toc_priority: 5 +toc_title: Kafka +--- + # Kafka {#kafka} Движок работает с [Apache Kafka](http://kafka.apache.org/). diff --git a/docs/ru/engines/table-engines/integrations/mysql.md b/docs/ru/engines/table-engines/integrations/mysql.md index 9891fe79959..3370e9b06d0 100644 --- a/docs/ru/engines/table-engines/integrations/mysql.md +++ b/docs/ru/engines/table-engines/integrations/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 3 +toc_title: MySQL +--- + # MySQL {#mysql} Движок MySQL позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом MySQL сервере. diff --git a/docs/ru/engines/table-engines/integrations/odbc.md b/docs/ru/engines/table-engines/integrations/odbc.md index aecdbbf038d..97317d647c8 100644 --- a/docs/ru/engines/table-engines/integrations/odbc.md +++ b/docs/ru/engines/table-engines/integrations/odbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 1 +toc_title: ODBC +--- + # ODBC {#table-engine-odbc} Позволяет ClickHouse подключаться к внешним базам данных с помощью [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index ef7b811e295..dedb5842d68 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -45,7 +45,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `rabbitmq_row_delimiter` – символ-разделитель, который завершает сообщение. - `rabbitmq_schema` – опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap’n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. - `rabbitmq_num_consumers` – количество потребителей на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. -- `rabbitmq_num_queues` – количество очередей на потребителя. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одной очереди на потребителя недостаточна. +- `rabbitmq_num_queues` – количество очередей. По умолчанию: `1`. Большее число очередей может сильно увеличить пропускную способность. - `rabbitmq_queue_base` - настройка для имен очередей. Сценарии использования описаны ниже. - `rabbitmq_persistent` - флаг, от которого зависит настройка 'durable' для сообщений при запросах `INSERT`. По умолчанию: `0`. - `rabbitmq_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `rabbitmq_skip_broken_messages = N`, то движок отбрасывает `N` сообщений, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. @@ -140,4 +140,5 @@ Example: - `_channel_id` - идентификатор канала `ChannelID`, на котором было получено сообщение. - `_delivery_tag` - значение `DeliveryTag` полученного сообщения. Уникально в рамках одного канала. - `_redelivered` - флаг `redelivered`. (Не равно нулю, если есть возможность, что сообщение было получено более, чем одним каналом.) -- `_message_id` - значение `MessageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. +- `_message_id` - значение поля `messageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. +- `_timestamp` - значение поля `timestamp` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. diff --git a/docs/ru/engines/table-engines/log-family/index.md b/docs/ru/engines/table-engines/log-family/index.md index d2cb0df5a8a..7c6d2f81d7c 100644 --- a/docs/ru/engines/table-engines/log-family/index.md +++ b/docs/ru/engines/table-engines/log-family/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: Семейство Log -toc_title: Введение +toc_folder_title: "\u0421\u0435\u043c\u0435\u0439\u0441\u0442\u0432\u043e\u0020\u004c\u006f\u0067" +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" toc_priority: 29 --- diff --git a/docs/ru/engines/table-engines/log-family/log.md b/docs/ru/engines/table-engines/log-family/log.md index 826738d6a3d..fad331454c7 100644 --- a/docs/ru/engines/table-engines/log-family/log.md +++ b/docs/ru/engines/table-engines/log-family/log.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: Log +--- + # Log {#log} Движок относится к семейству движков Log. Смотрите общие свойства и различия движков в статье [Семейство Log](index.md). diff --git a/docs/ru/engines/table-engines/log-family/stripelog.md b/docs/ru/engines/table-engines/log-family/stripelog.md index 9523a354ee7..e505aae4c52 100644 --- a/docs/ru/engines/table-engines/log-family/stripelog.md +++ b/docs/ru/engines/table-engines/log-family/stripelog.md @@ -1,3 +1,8 @@ +--- +toc_priority: 32 +toc_title: StripeLog +--- + # StripeLog {#stripelog} Движок относится к семейству движков Log. Смотрите общие свойства и различия движков в статье [Семейство Log](index.md). diff --git a/docs/ru/engines/table-engines/log-family/tinylog.md b/docs/ru/engines/table-engines/log-family/tinylog.md index 69bc88b133e..d5c24d41ca4 100644 --- a/docs/ru/engines/table-engines/log-family/tinylog.md +++ b/docs/ru/engines/table-engines/log-family/tinylog.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: TinyLog +--- + # TinyLog {#tinylog} Движок относится к семейству движков Log. Смотрите общие свойства и различия движков в статье [Семейство Log](index.md). diff --git a/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md index 3e18f890af7..99b4ec06765 100644 --- a/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: AggregatingMergeTree +--- + # AggregatingMergeTree {#aggregatingmergetree} Движок наследует функциональность [MergeTree](mergetree.md#table_engines-mergetree), изменяя логику слияния кусков данных. Все строки с одинаковым первичным ключом (точнее, с одинаковым [ключом сортировки](mergetree.md)) ClickHouse заменяет на одну (в пределах одного куска данных), которая хранит объединение состояний агрегатных функций. diff --git a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md index f7bc14cc647..8ea3a5a7c92 100644 --- a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: CollapsingMergeTree +--- + # CollapsingMergeTree {#table_engine-collapsingmergetree} Движок наследует функциональность от [MergeTree](mergetree.md) и добавляет в алгоритм слияния кусков данных логику сворачивания (удаления) строк. diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 32b5eee7fab..2d26528d964 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,3 +1,9 @@ +--- +toc_priority: 32 +toc_title: "\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u043b\u044c\u043d\u044b\u0439\u0020\u043a\u043b\u044e\u0447\u0020\u043f\u0430\u0440\u0442\u0438\u0446\u0438\u043e\u043d\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +--- + + # Произвольный ключ партиционирования {#proizvolnyi-kliuch-partitsionirovaniia} Партиционирование данных доступно для таблиц семейства [MergeTree](mergetree.md) (включая [реплицированные таблицы](replication.md)). Таблицы [MaterializedView](../special/materializedview.md#materializedview), созданные на основе таблиц MergeTree, также поддерживают партиционирование. diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index 861a8b6e633..e47c9127711 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: GraphiteMergeTree +--- + # GraphiteMergeTree {#graphitemergetree} Движок предназначен для прореживания и агрегирования/усреднения (rollup) данных [Graphite](http://graphite.readthedocs.io/en/latest/index.html). Он может быть интересен разработчикам, которые хотят использовать ClickHouse как хранилище данных для Graphite. diff --git a/docs/ru/engines/table-engines/mergetree-family/index.md b/docs/ru/engines/table-engines/mergetree-family/index.md index 9e989d807da..abdfdd77d7f 100644 --- a/docs/ru/engines/table-engines/mergetree-family/index.md +++ b/docs/ru/engines/table-engines/mergetree-family/index.md @@ -1,6 +1,5 @@ --- toc_folder_title: MergeTree Family toc_priority: 28 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" --- - - diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md index 4aa1eb556f3..1228371e8ea 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: ReplacingMergeTree +--- + # ReplacingMergeTree {#replacingmergetree} Движок отличается от [MergeTree](mergetree.md#table_engines-mergetree) тем, что выполняет удаление дублирующихся записей с одинаковым значением [ключа сортировки](mergetree.md)). diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 410cf06c0c5..426c456ba3a 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -1,3 +1,8 @@ +--- +toc_priority: 31 +toc_title: "\u0420\u0435\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u044f\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +--- + # Репликация данных {#table_engines-replication} Репликация поддерживается только для таблиц семейства MergeTree: diff --git a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md index 070cd109c30..7b9c11adc2e 100644 --- a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: SummingMergeTree +--- + # SummingMergeTree {#summingmergetree} Движок наследует функциональность [MergeTree](mergetree.md#table_engines-mergetree). Отличие заключается в том, что для таблиц `SummingMergeTree` при слиянии кусков данных ClickHouse все строки с одинаковым первичным ключом (точнее, с одинаковым [ключом сортировки](mergetree.md)) заменяет на одну, которая хранит только суммы значений из столбцов с цифровым типом данных. Если ключ сортировки подобран таким образом, что одному значению ключа соответствует много строк, это значительно уменьшает объём хранения и ускоряет последующую выборку данных. diff --git a/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index bf280eb52bc..2adb8cc0d77 100644 --- a/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: VersionedCollapsingMergeTree +--- + # VersionedCollapsingMergeTree {#versionedcollapsingmergetree} Движок: diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index 72d2822af98..986fe9adbb9 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: Buffer +--- + # Buffer {#buffer} Буферизует записываемые данные в оперативке, периодически сбрасывая их в другую таблицу. При чтении, производится чтение данных одновременно из буфера и из другой таблицы. diff --git a/docs/ru/engines/table-engines/special/dictionary.md b/docs/ru/engines/table-engines/special/dictionary.md index b35c8cecc8f..048da157b2d 100644 --- a/docs/ru/engines/table-engines/special/dictionary.md +++ b/docs/ru/engines/table-engines/special/dictionary.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: Dictionary +--- + # Dictionary {#dictionary} Движок `Dictionary` отображает данные [словаря](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) как таблицу ClickHouse. diff --git a/docs/ru/engines/table-engines/special/distributed.md b/docs/ru/engines/table-engines/special/distributed.md index c8f7fe6eba7..7ab0b916337 100644 --- a/docs/ru/engines/table-engines/special/distributed.md +++ b/docs/ru/engines/table-engines/special/distributed.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: Distributed +--- + # Distributed {#distributed} **Движок Distributed не хранит данные самостоятельно**, а позволяет обрабатывать запросы распределённо, на нескольких серверах. Чтение автоматически распараллеливается. При чтении будут использованы индексы таблиц на удалённых серверах, если есть. diff --git a/docs/ru/engines/table-engines/special/external-data.md b/docs/ru/engines/table-engines/special/external-data.md index 41708978367..7e383c0c12d 100644 --- a/docs/ru/engines/table-engines/special/external-data.md +++ b/docs/ru/engines/table-engines/special/external-data.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0435\u0020\u0434\u043b\u044f\u0020\u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u0430" +--- + # Внешние данные для обработки запроса {#vneshnie-dannye-dlia-obrabotki-zaprosa} ClickHouse позволяет отправить на сервер данные, необходимые для обработки одного запроса, вместе с запросом SELECT. Такие данные будут положены во временную таблицу (см. раздел «Временные таблицы») и смогут использоваться в запросе (например, в операторах IN). diff --git a/docs/ru/engines/table-engines/special/file.md b/docs/ru/engines/table-engines/special/file.md index 907988a47aa..6a55ef31732 100644 --- a/docs/ru/engines/table-engines/special/file.md +++ b/docs/ru/engines/table-engines/special/file.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: File +--- + # File(Format) {#table_engines-file} Управляет данными в одном файле на диске в указанном формате. diff --git a/docs/ru/engines/table-engines/special/index.md b/docs/ru/engines/table-engines/special/index.md index dcf46c503b2..0d86461dd2d 100644 --- a/docs/ru/engines/table-engines/special/index.md +++ b/docs/ru/engines/table-engines/special/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: Special +toc_folder_title: "\u0421\u043f\u0435\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0435\u0020\u0434\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446" toc_priority: 31 --- diff --git a/docs/ru/engines/table-engines/special/join.md b/docs/ru/engines/table-engines/special/join.md index aa734f5ca55..65bece4724c 100644 --- a/docs/ru/engines/table-engines/special/join.md +++ b/docs/ru/engines/table-engines/special/join.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: Join +--- + # Join {#join} Подготовленная структура данных для использования в операциях [JOIN](../../../engines/table-engines/special/join.md#select-join). diff --git a/docs/ru/engines/table-engines/special/materializedview.md b/docs/ru/engines/table-engines/special/materializedview.md index ae7a9965a24..1281d1db9ab 100644 --- a/docs/ru/engines/table-engines/special/materializedview.md +++ b/docs/ru/engines/table-engines/special/materializedview.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: MaterializedView +--- + # MaterializedView {#materializedview} Используется для реализации материализованных представлений (подробнее см. запрос [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query)). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок. diff --git a/docs/ru/engines/table-engines/special/memory.md b/docs/ru/engines/table-engines/special/memory.md index 6de7978d134..9ca189ef3b2 100644 --- a/docs/ru/engines/table-engines/special/memory.md +++ b/docs/ru/engines/table-engines/special/memory.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: Memory +--- + # Memory {#memory} Хранит данные в оперативке, в несжатом виде. Данные хранятся именно в таком виде, в каком они получаются при чтении. То есть, само чтение из этой таблицы полностью бесплатно. diff --git a/docs/ru/engines/table-engines/special/merge.md b/docs/ru/engines/table-engines/special/merge.md index 4f6e1751591..656aa7cfd6b 100644 --- a/docs/ru/engines/table-engines/special/merge.md +++ b/docs/ru/engines/table-engines/special/merge.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: Merge +--- + # Merge {#merge} Движок `Merge` (не путайте с движком `MergeTree`) не хранит данные самостоятельно, а позволяет читать одновременно из произвольного количества других таблиц. diff --git a/docs/ru/engines/table-engines/special/null.md b/docs/ru/engines/table-engines/special/null.md index ac8c7942f79..2c3af1ce11e 100644 --- a/docs/ru/engines/table-engines/special/null.md +++ b/docs/ru/engines/table-engines/special/null.md @@ -1,4 +1,7 @@ -# Null {#null} +--- +toc_priority: 38 +toc_title: 'Null' +--- При записи в таблицу типа Null, данные игнорируются. При чтении из таблицы типа Null, возвращается пустота. diff --git a/docs/ru/engines/table-engines/special/set.md b/docs/ru/engines/table-engines/special/set.md index 497d52821d1..73fe3c3cf50 100644 --- a/docs/ru/engines/table-engines/special/set.md +++ b/docs/ru/engines/table-engines/special/set.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: Set +--- + # Set {#set} Представляет собой множество, постоянно находящееся в оперативке. Предназначено для использования в правой части оператора IN (смотрите раздел «Операторы IN»). diff --git a/docs/ru/engines/table-engines/special/url.md b/docs/ru/engines/table-engines/special/url.md index 69c1567a1c2..cdb5afddf75 100644 --- a/docs/ru/engines/table-engines/special/url.md +++ b/docs/ru/engines/table-engines/special/url.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: URL +--- + # URL(URL, Format) {#table_engines-url} Управляет данными на удаленном HTTP/HTTPS сервере. Данный движок похож diff --git a/docs/ru/engines/table-engines/special/view.md b/docs/ru/engines/table-engines/special/view.md index a732a675d4c..18813a55da2 100644 --- a/docs/ru/engines/table-engines/special/view.md +++ b/docs/ru/engines/table-engines/special/view.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: View +--- + # View {#table_engines-view} Используется для реализации представлений (подробнее см. запрос `CREATE VIEW`). Не хранит данные, а хранит только указанный запрос `SELECT`. При чтении из таблицы, выполняет его (с удалением из запроса всех ненужных столбцов). diff --git a/docs/ru/faq/index.md b/docs/ru/faq/index.md index 2ee9d51e83b..d9eccf59c5c 100644 --- a/docs/ru/faq/index.md +++ b/docs/ru/faq/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: F.A.Q. +toc_hidden: true toc_priority: 76 --- - diff --git a/docs/ru/getting-started/example-datasets/amplab-benchmark.md b/docs/ru/getting-started/example-datasets/amplab-benchmark.md index b503fc909f3..bc59672ab26 100644 --- a/docs/ru/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/ru/getting-started/example-datasets/amplab-benchmark.md @@ -1,3 +1,8 @@ +--- +toc_priority: 19 +toc_title: AMPLab Big Data Benchmark +--- + # AMPLab Big Data Benchmark {#amplab-big-data-benchmark} См. https://amplab.cs.berkeley.edu/benchmark/ diff --git a/docs/ru/getting-started/example-datasets/criteo.md b/docs/ru/getting-started/example-datasets/criteo.md index 4abb8c43f35..7a58da5b695 100644 --- a/docs/ru/getting-started/example-datasets/criteo.md +++ b/docs/ru/getting-started/example-datasets/criteo.md @@ -1,3 +1,8 @@ +--- +toc_priority: 18 +toc_title: "\u0422\u0435\u0440\u0430\u0431\u0430\u0439\u0442\u0020\u043b\u043e\u0433\u043e\u0432\u0020\u043a\u043b\u0438\u043a\u043e\u0432\u0020\u043e\u0442\u0020\u0043\u0072\u0069\u0074\u0065\u006f" +--- + # Терабайт логов кликов от Criteo {#terabait-logov-klikov-ot-criteo} Скачайте данные с http://labs.criteo.com/downloads/download-terabyte-click-logs/ diff --git a/docs/ru/getting-started/example-datasets/index.md b/docs/ru/getting-started/example-datasets/index.md index c349e369b1d..eff944a7980 100644 --- a/docs/ru/getting-started/example-datasets/index.md +++ b/docs/ru/getting-started/example-datasets/index.md @@ -1,8 +1,7 @@ --- -toc_folder_title: "\u0422\u0435\u0441\u0442\u043E\u0432\u044B\u0435 \u043C\u0430\u0441\ - \u0441\u0438\u0432\u044B \u0434\u0430\u043D\u043D\u044B\u0445" -toc_priority: 12 -toc_title: "\u041E\u0431\u0437\u043E\u0440" +toc_folder_title: "\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0435\u0020\u043c\u0430\u0441\u0441\u0438\u0432\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_priority: 14 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" --- # Тестовые массивы данных {#testovye-massivy-dannykh} diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index ab2fe3d6c64..e8a3163376c 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -1,3 +1,8 @@ +--- +toc_priority: 15 +toc_title: "\u0410\u043d\u043e\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0435\u0020\u042f\u043d\u0434\u0435\u043a\u0441\u002e\u041c\u0435\u0442\u0440\u0438\u043a\u0438" +--- + # Анонимизированные данные Яндекс.Метрики {#anonimizirovannye-dannye-iandeks-metriki} Датасет состоит из двух таблиц, содержащих анонимизированные данные о хитах (`hits_v1`) и визитах (`visits_v1`) Яндекс.Метрики. Каждую из таблиц можно скачать в виде сжатого `.tsv.xz`-файла или в виде уже готовых партиций. Также можно скачать расширенную версию таблицы `hits`, содержащую 100 миллионов строк в виде [архива c файлами TSV](https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_100m_obfuscated_v1.tsv.xz) и в виде [готовых партиций](https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz). diff --git a/docs/ru/getting-started/example-datasets/nyc-taxi.md b/docs/ru/getting-started/example-datasets/nyc-taxi.md index 64202b2e765..1f981324261 100644 --- a/docs/ru/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ru/getting-started/example-datasets/nyc-taxi.md @@ -1,3 +1,8 @@ +--- +toc_priority: 20 +toc_title: "\u0414\u0430\u043d\u043d\u044b\u0435\u0020\u043e\u0020\u0442\u0430\u043a\u0441\u0438\u0020\u0432\u0020\u041d\u044c\u044e\u002d\u0419\u043e\u0440\u043a\u0435" +--- + # Данные о такси в Нью-Йорке {#dannye-o-taksi-v-niu-iorke} Этот датасет может быть получен двумя способами: diff --git a/docs/ru/getting-started/example-datasets/ontime.md b/docs/ru/getting-started/example-datasets/ontime.md index 7e7a6be2196..4d3eea14da6 100644 --- a/docs/ru/getting-started/example-datasets/ontime.md +++ b/docs/ru/getting-started/example-datasets/ontime.md @@ -1,3 +1,8 @@ +--- +toc_priority: 21 +toc_title: OnTime +--- + # OnTime {#ontime} Этот датасет может быть получен двумя способами: diff --git a/docs/ru/getting-started/example-datasets/star-schema.md b/docs/ru/getting-started/example-datasets/star-schema.md index d938b1e8620..f7502e8409d 100644 --- a/docs/ru/getting-started/example-datasets/star-schema.md +++ b/docs/ru/getting-started/example-datasets/star-schema.md @@ -1,3 +1,8 @@ +--- +toc_priority: 16 +toc_title: Star Schema Benchmark +--- + # Star Schema Benchmark {#star-schema-benchmark} Компиляция dbgen: diff --git a/docs/ru/getting-started/example-datasets/wikistat.md b/docs/ru/getting-started/example-datasets/wikistat.md index 0e3e269fe9f..c5a877ff8fd 100644 --- a/docs/ru/getting-started/example-datasets/wikistat.md +++ b/docs/ru/getting-started/example-datasets/wikistat.md @@ -1,3 +1,8 @@ +--- +toc_priority: 17 +toc_title: WikiStat +--- + # WikiStat {#wikistat} См: http://dumps.wikimedia.org/other/pagecounts-raw/ diff --git a/docs/ru/getting-started/index.md b/docs/ru/getting-started/index.md index b2b292c4725..ab72ce4a1d2 100644 --- a/docs/ru/getting-started/index.md +++ b/docs/ru/getting-started/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "\u041D\u0430\u0447\u0430\u043B\u043E \u0440\u0430\u0431\u043E\u0442\ - \u044B" +toc_folder_title: "\u041d\u0430\u0447\u0430\u043b\u043e\u0020\u0440\u0430\u0431\u043e\u0442\u044b" toc_hidden: true toc_priority: 8 toc_title: hidden diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 3e9270ef3bc..fb14ecfe599 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -1,3 +1,8 @@ +--- +toc_priority: 11 +toc_title: "\u0423\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0430" +--- + # Установка {#ustanovka} ## Системные требования {#sistemnye-trebovaniia} diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md index e3bae5c9756..86a5cd5272c 100644 --- a/docs/ru/getting-started/playground.md +++ b/docs/ru/getting-started/playground.md @@ -1,3 +1,8 @@ +--- +toc_priority: 14 +toc_title: Playground +--- + # ClickHouse Playground {#clickhouse-playground} [ClickHouse Playground](https://play.clickhouse.tech) позволяет пользователям экспериментировать с ClickHouse, мгновенно выполняя запросы без настройки своего сервера или кластера. diff --git a/docs/ru/guides/apply-catboost-model.md b/docs/ru/guides/apply-catboost-model.md index 3515d2731c2..026b4d9d75e 100644 --- a/docs/ru/guides/apply-catboost-model.md +++ b/docs/ru/guides/apply-catboost-model.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: "\u041f\u0440\u0438\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u0020\u043c\u043e\u0434\u0435\u043b\u0438\u0020\u0043\u0061\u0074\u0042\u006f\u006f\u0073\u0074\u0020\u0432\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Применение модели CatBoost в ClickHouse {#applying-catboost-model-in-clickhouse} [CatBoost](https://catboost.ai) — открытая программная библиотека разработанная компанией [Яндекс](https://yandex.ru/company/) для машинного обучения, которая использует схему градиентного бустинга. diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 816b5962280..222af33f952 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -1,3 +1,8 @@ +--- +toc_priority: 17 +toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0020\u043a\u043e\u043c\u0430\u043d\u0434\u043d\u043e\u0439\u0020\u0441\u0442\u0440\u043e\u043a\u0438" +--- + # Клиент командной строки {#klient-komandnoi-stroki} ClickHouse предоставляет собственный клиент командной строки: `clickhouse-client`. Клиент поддерживает запуск с аргументами командной строки и с конфигурационными файлами. Подробнее читайте в разделе [Конфигурирование](#interfaces_cli_configuration). diff --git a/docs/ru/interfaces/cpp.md b/docs/ru/interfaces/cpp.md index 7136af3197b..264b4f82500 100644 --- a/docs/ru/interfaces/cpp.md +++ b/docs/ru/interfaces/cpp.md @@ -1,3 +1,8 @@ +--- +toc_priority: 24 +toc_title: "\u0043\u002b\u002b\u0020\u043a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0430\u044f\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0430" +--- + # C++ клиентская библиотека {#c-klientskaia-biblioteka} См. README в репозитории [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp). diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 2745139998f..042c62e310c 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1,44 +1,61 @@ +--- +toc_priority: 21 +toc_title: "\u0424\u043e\u0440\u043c\u0430\u0442\u044b\u0020\u0432\u0445\u043e\u0434\u043d\u044b\u0445\u0020\u0438\u0020\u0432\u044b\u0445\u043e\u0434\u043d\u044b\u0445\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +--- + # Форматы входных и выходных данных {#formats} ClickHouse может принимать (`INSERT`) и отдавать (`SELECT`) данные в различных форматах. Поддерживаемые форматы и возможность использовать их в запросах `INSERT` и `SELECT` перечислены в таблице ниже. -| Формат | INSERT | SELECT | -|-----------------------------------------------------------------|--------|--------| -| [TabSeparated](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | -| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | -| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | -| [Template](#format-template) | ✔ | ✔ | -| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | -| [CSV](#csv) | ✔ | ✔ | -| [CSVWithNames](#csvwithnames) | ✔ | ✔ | -| [CustomSeparated](#format-customseparated) | ✔ | ✔ | -| [Values](#data-format-values) | ✔ | ✔ | -| [Vertical](#vertical) | ✗ | ✔ | -| [JSON](#json) | ✗ | ✔ | -| [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | -| [TSKV](#tskv) | ✔ | ✔ | -| [Pretty](#pretty) | ✗ | ✔ | -| [PrettyCompact](#prettycompact) | ✗ | ✔ | -| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | -| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | -| [PrettySpace](#prettyspace) | ✗ | ✔ | -| [Protobuf](#protobuf) | ✔ | ✔ | -| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | -| [Parquet](#data-format-parquet) | ✔ | ✔ | -| [Arrow](#data-format-arrow) | ✔ | ✔ | -| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✗ | -| [RowBinary](#rowbinary) | ✔ | ✔ | -| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [Native](#native) | ✔ | ✔ | -| [Null](#null) | ✗ | ✔ | -| [XML](#xml) | ✗ | ✔ | -| [CapnProto](#capnproto) | ✔ | ✗ | -| [LineAsString](#lineasstring) | ✔ | ✗ | +| Формат | INSERT | SELECT | +|-----------------------------------------------------------------------------------------|--------|--------| +| [TabSeparated](#tabseparated) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | +| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | +| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | +| [Template](#format-template) | ✔ | ✔ | +| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | +| [CSV](#csv) | ✔ | ✔ | +| [CSVWithNames](#csvwithnames) | ✔ | ✔ | +| [CustomSeparated](#format-customseparated) | ✔ | ✔ | +| [Values](#data-format-values) | ✔ | ✔ | +| [Vertical](#vertical) | ✗ | ✔ | +| [VerticalRaw](#verticalraw) | ✗ | ✔ | +| [JSON](#json) | ✗ | ✔ | +| [JSONString](#jsonstring) | ✗ | ✔ | +| [JSONCompact](#jsoncompact) | ✗ | ✔ | +| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | +| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | +| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | +| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ | +| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ | +| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | +| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | +| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ | +| [TSKV](#tskv) | ✔ | ✔ | +| [Pretty](#pretty) | ✗ | ✔ | +| [PrettyCompact](#prettycompact) | ✗ | ✔ | +| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | +| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | +| [PrettySpace](#prettyspace) | ✗ | ✔ | +| [Protobuf](#protobuf) | ✔ | ✔ | +| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | +| [Avro](#data-format-avro) | ✔ | ✔ | +| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | +| [Parquet](#data-format-parquet) | ✔ | ✔ | +| [Arrow](#data-format-arrow) | ✔ | ✔ | +| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✗ | +| [RowBinary](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [Native](#native) | ✔ | ✔ | +| [Null](#null) | ✗ | ✔ | +| [XML](#xml) | ✗ | ✔ | +| [CapnProto](#capnproto) | ✔ | ✗ | +| [LineAsString](#lineasstring) | ✔ | ✗ | Вы можете регулировать некоторые параметры работы с форматами с помощью настроек ClickHouse. За дополнительной информацией обращайтесь к разделу [Настройки](../operations/settings/settings.md). @@ -364,62 +381,41 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA "meta": [ { - "name": "SearchPhrase", + "name": "'hello'", "type": "String" }, { - "name": "c", + "name": "multiply(42, number)", "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" } ], "data": [ { - "SearchPhrase": "", - "c": "8267016" + "'hello'": "hello", + "multiply(42, number)": "0", + "range(5)": [0,1,2,3,4] }, { - "SearchPhrase": "bathroom interior design", - "c": "2166" + "'hello'": "hello", + "multiply(42, number)": "42", + "range(5)": [0,1,2,3,4] }, { - "SearchPhrase": "yandex", - "c": "1655" - }, - { - "SearchPhrase": "spring 2014 fashion", - "c": "1549" - }, - { - "SearchPhrase": "freeform photos", - "c": "1480" + "'hello'": "hello", + "multiply(42, number)": "84", + "range(5)": [0,1,2,3,4] } ], - "totals": - { - "SearchPhrase": "", - "c": "8873898" - }, + "rows": 3, - "extremes": - { - "min": - { - "SearchPhrase": "", - "c": "1480" - }, - "max": - { - "SearchPhrase": "", - "c": "8267016" - } - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 + "rows_before_limit_at_least": 3 } ``` @@ -438,65 +434,167 @@ JSON совместим с JavaScript. Для этого, дополнитель ClickHouse поддерживает [NULL](../sql-reference/syntax.md), который при выводе JSON будет отображен как `null`. Чтобы включить отображение в результате значений `+nan`, `-nan`, `+inf`, `-inf`, установите параметр [output_format_json_quote_denormals](../operations/settings/settings.md#settings-output_format_json_quote_denormals) равным 1. -Смотрите также формат [JSONEachRow](#jsoneachrow) . +Смотрите также формат [JSONEachRow](#jsoneachrow). + +## JSONString {#jsonstring} + +Отличается от JSON только тем, что поля данных выводятся в строках, а не в типизированных значениях JSON. + +Пример: + +```json +{ + "meta": + [ + { + "name": "'hello'", + "type": "String" + }, + { + "name": "multiply(42, number)", + "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" + } + ], + + "data": + [ + { + "'hello'": "hello", + "multiply(42, number)": "0", + "range(5)": "[0,1,2,3,4]" + }, + { + "'hello'": "hello", + "multiply(42, number)": "42", + "range(5)": "[0,1,2,3,4]" + }, + { + "'hello'": "hello", + "multiply(42, number)": "84", + "range(5)": "[0,1,2,3,4]" + } + ], + + "rows": 3, + + "rows_before_limit_at_least": 3 +} +``` ## JSONCompact {#jsoncompact} +## JSONCompactString {#jsoncompactstring} Отличается от JSON только тем, что строчки данных выводятся в массивах, а не в object-ах. Пример: ``` json +// JSONCompact { "meta": [ { - "name": "SearchPhrase", + "name": "'hello'", "type": "String" }, { - "name": "c", + "name": "multiply(42, number)", "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" } ], "data": [ - ["", "8267016"], - ["интерьер ванной комнаты", "2166"], - ["яндекс", "1655"], - ["весна 2014 мода", "1549"], - ["фриформ фото", "1480"] + ["hello", "0", [0,1,2,3,4]], + ["hello", "42", [0,1,2,3,4]], + ["hello", "84", [0,1,2,3,4]] ], - "totals": ["","8873898"], + "rows": 3, - "extremes": - { - "min": ["","1480"], - "max": ["","8267016"] - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 + "rows_before_limit_at_least": 3 } ``` -Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). -Смотрите также формат `JSONEachRow`. +```json +// JSONCompactString +{ + "meta": + [ + { + "name": "'hello'", + "type": "String" + }, + { + "name": "multiply(42, number)", + "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" + } + ], -## JSONEachRow {#jsoneachrow} + "data": + [ + ["hello", "0", "[0,1,2,3,4]"], + ["hello", "42", "[0,1,2,3,4]"], + ["hello", "84", "[0,1,2,3,4]"] + ], -При использовании этого формата, ClickHouse выводит каждую запись как объект JSON (каждый объект отдельной строкой), при этом данные в целом — невалидный JSON. + "rows": 3, -``` json -{"SearchPhrase":"дизайн штор","count()":"1064"} -{"SearchPhrase":"баку","count()":"1000"} -{"SearchPhrase":"","count":"8267016"} + "rows_before_limit_at_least": 3 +} ``` -При вставке данных необходимо каждую запись передавать как отдельный объект JSON. +## JSONEachRow {#jsoneachrow} +## JSONStringEachRow {#jsonstringeachrow} +## JSONCompactEachRow {#jsoncompacteachrow} +## JSONCompactStringEachRow {#jsoncompactstringeachrow} + +При использовании этих форматов ClickHouse выводит каждую запись как значения JSON (каждое значение отдельной строкой), при этом данные в целом — невалидный JSON. + +``` json +{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow +[42,"hello",[1,"a"]] // JSONCompactEachRow +["42","hello","(2,'a')"] // JSONCompactStringsEachRow +``` + +При вставке данных вы должны предоставить отдельное значение JSON для каждой строки. + +## JSONEachRowWithProgress {#jsoneachrowwithprogress} +## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} + +Отличается от `JSONEachRow`/`JSONStringEachRow` тем, что ClickHouse будет выдавать информацию о ходе выполнения в виде значений JSON. + +```json +{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} +{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}} +{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}} +{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}} +``` + +## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} +## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} + +Отличается от `JSONCompactEachRow`/`JSONCompactStringEachRow` тем, что имена и типы столбцов записываются как первые две строки. + +```json +["'hello'", "multiply(42, number)", "range(5)"] +["String", "UInt64", "Array(UInt8)"] +["hello", "0", [0,1,2,3,4]] +["hello", "42", [0,1,2,3,4]] +["hello", "84", [0,1,2,3,4]] +``` ### Вставка данных {#vstavka-dannykh} @@ -784,6 +882,10 @@ test: string with 'quotes' and with some special Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). +## VerticalRaw {#verticalraw} + +Аналогичен [Vertical](#vertical), но с отключенным выходом. Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). + ## XML {#xml} Формат XML подходит только для вывода данных, не для парсинга. Пример: diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index ffe4b2e5276..279f2916c78 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -1,3 +1,8 @@ +--- +toc_priority: 19 +toc_title: "\u0048\u0054\u0054\u0050\u002d\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441" +--- + # HTTP-интерфейс {#http-interface} HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md index e83cd2cfacf..196dba64933 100644 --- a/docs/ru/interfaces/jdbc.md +++ b/docs/ru/interfaces/jdbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 22 +toc_title: "\u004a\u0044\u0042\u0043\u002d\u0434\u0440\u0430\u0439\u0432\u0435\u0440" +--- + # JDBC-драйвер {#jdbc-draiver} - **[Официальный драйвер](https://github.com/ClickHouse/clickhouse-jdbc)** diff --git a/docs/ru/interfaces/mysql.md b/docs/ru/interfaces/mysql.md index 79368aa1ba2..fa0003e0bea 100644 --- a/docs/ru/interfaces/mysql.md +++ b/docs/ru/interfaces/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 20 +toc_title: "\u004d\u0079\u0053\u0051\u004c\u002d\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441" +--- + # MySQL-интерфейс {#mysql-interface} ClickHouse поддерживает взаимодействие по протоколу MySQL. Данная функция включается настройкой [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) в конфигурационном файле: diff --git a/docs/ru/interfaces/odbc.md b/docs/ru/interfaces/odbc.md index 0f4a6279539..728c4bd6979 100644 --- a/docs/ru/interfaces/odbc.md +++ b/docs/ru/interfaces/odbc.md @@ -1,3 +1,9 @@ +--- +toc_priority: 23 +toc_title: "\u004f\u0044\u0042\u0043\u002d\u0434\u0440\u0430\u0439\u0432\u0435\u0440" +--- + + # ODBC-драйвер {#odbc-draiver} - [Официальный драйвер](https://github.com/ClickHouse/clickhouse-odbc). diff --git a/docs/ru/interfaces/tcp.md b/docs/ru/interfaces/tcp.md index 6bdfb286ac2..d89646f15b7 100644 --- a/docs/ru/interfaces/tcp.md +++ b/docs/ru/interfaces/tcp.md @@ -1,3 +1,8 @@ +--- +toc_priority: 18 +toc_title: "\u0420\u043e\u0434\u043d\u043e\u0439\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0020\u0028\u0054\u0043\u0050\u0029" +--- + # Родной интерфейс (TCP) {#rodnoi-interfeis-tcp} Нативный протокол используется в [клиенте командной строки](cli.md), для взаимодействия между серверами во время обработки распределенных запросов, а также в других программах на C++. К сожалению, у родного протокола ClickHouse пока нет формальной спецификации, но в нем можно разобраться с использованием исходного кода ClickHouse (начиная с [примерно этого места](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) и/или путем перехвата и анализа TCP трафика. diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 66a2252b23a..f35acb9e968 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -1,3 +1,8 @@ +--- +toc_priority: 26 +toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Клиентские библиотеки от сторонних разработчиков {#klientskie-biblioteki-ot-storonnikh-razrabotchikov} !!! warning "Disclaimer" diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index f7eaa5cc77f..1fabdb8a31c 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -1,3 +1,9 @@ +--- +toc_priority: 28 +toc_title: "\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u044b\u0435\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044b\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + + # Визуальные интерфейсы от сторонних разработчиков {#vizualnye-interfeisy-ot-storonnikh-razrabotchikov} ## С открытым исходным кодом {#s-otkrytym-iskhodnym-kodom} diff --git a/docs/ru/interfaces/third-party/index.md b/docs/ru/interfaces/third-party/index.md index bfdd554aebd..a57169df73b 100644 --- a/docs/ru/interfaces/third-party/index.md +++ b/docs/ru/interfaces/third-party/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "\u041E\u0442 \u0441\u0442\u043E\u0440\u043E\u043D\u043D\u0438\u0445\ - \ \u0440\u0430\u0437\u0440\u0430\u0431\u043E\u0442\u0447\u0438\u043A\u043E\u0432" +toc_folder_title: "\u0421\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0435\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044b" toc_priority: 24 --- diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 527f65343db..60d6181ab3f 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -1,3 +1,8 @@ +--- +toc_priority: 27 +toc_title: "\u0411\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438\u0020\u0434\u043b\u044f\u0020\u0438\u043d\u0442\u0435\u0433\u0440\u0430\u0446\u0438\u0438\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Библиотеки для интеграции от сторонних разработчиков {#biblioteki-dlia-integratsii-ot-storonnikh-razrabotchikov} !!! warning "Disclaimer" diff --git a/docs/ru/interfaces/third-party/proxy.md b/docs/ru/interfaces/third-party/proxy.md index 225c3fee150..fc66ecde293 100644 --- a/docs/ru/interfaces/third-party/proxy.md +++ b/docs/ru/interfaces/third-party/proxy.md @@ -1,3 +1,8 @@ +--- +toc_priority: 29 +toc_title: "\u041f\u0440\u043e\u043a\u0441\u0438\u002d\u0441\u0435\u0440\u0432\u0435\u0440\u044b\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Прокси-серверы от сторонних разработчиков {#proksi-servery-ot-storonnikh-razrabotchikov} ## chproxy {#chproxy} diff --git a/docs/ru/introduction/distinctive-features.md b/docs/ru/introduction/distinctive-features.md index 98dbe6df134..4eeeef4a443 100644 --- a/docs/ru/introduction/distinctive-features.md +++ b/docs/ru/introduction/distinctive-features.md @@ -1,3 +1,8 @@ +--- +toc_priority: 4 +toc_title: "\u041e\u0442\u043b\u0438\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435\u0020\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Отличительные возможности ClickHouse {#otlichitelnye-vozmozhnosti-clickhouse} ## По-настоящему столбцовая СУБД {#po-nastoiashchemu-stolbtsovaia-subd} diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md index 65254b0f4f0..ab740954bbe 100644 --- a/docs/ru/introduction/history.md +++ b/docs/ru/introduction/history.md @@ -1,3 +1,9 @@ +--- +toc_priority: 7 +toc_title: "\u0418\u0441\u0442\u043e\u0440\u0438\u044f\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + + # История ClickHouse {#istoriia-clickhouse} ClickHouse изначально разрабатывался для обеспечения работы [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики, и продолжает быть её ключевым компонентом. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Данная статья вкратце демонстрирует какие цели исторически стояли перед ClickHouse на ранних этапах его развития. diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md index cbefa10cf7c..c449e76a6ea 100644 --- a/docs/ru/introduction/performance.md +++ b/docs/ru/introduction/performance.md @@ -1,3 +1,8 @@ +--- +toc_priority: 6 +toc_title: "\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c" +--- + # Производительность {#proizvoditelnost} По результатам внутреннего тестирования в Яндексе, ClickHouse обладает наиболее высокой производительностью (как наиболее высокой пропускной способностью на длинных запросах, так и наиболее низкой задержкой на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на [отдельной странице](https://clickhouse.tech/benchmark/dbms/). diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index 27dbc2fbf62..9973de91161 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -1,3 +1,8 @@ +--- +toc_priority: 48 +toc_title: "\u0423\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0434\u043e\u0441\u0442\u0443\u043f\u043e\u043c" +--- + # Управление доступом {#access-control} ClickHouse поддерживает управление доступом на основе подхода [RBAC](https://ru.wikipedia.org/wiki/Управление_доступом_на_основе_ролей). diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md index 89fb0403543..0dcb6fd307d 100644 --- a/docs/ru/operations/backup.md +++ b/docs/ru/operations/backup.md @@ -1,3 +1,8 @@ +--- +toc_priority: 49 +toc_title: "\u0420\u0435\u0437\u0435\u0440\u0432\u043d\u043e\u0435\u0020\u043a\u043e\u043f\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +--- + # Резервное копирование данных {#rezervnoe-kopirovanie-dannykh} [Репликация](../engines/table-engines/mergetree-family/replication.md) обеспечивает защиту от аппаратных сбоев, но не защищает от человеческих ошибок: случайного удаления данных, удаления не той таблицы, которую надо было, или таблицы на не том кластере, а также программных ошибок, которые приводят к неправильной обработке данных или их повреждению. Во многих случаях подобные ошибки влияют на все реплики. ClickHouse имеет встроенные средства защиты для предотвращения некоторых типов ошибок — например, по умолчанию [не получится удалить таблицы \*MergeTree, содержащие более 50 Гб данных, одной командой](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Однако эти средства защиты не охватывают все возможные случаи и могут быть обойдены. diff --git a/docs/ru/operations/configuration-files.md b/docs/ru/operations/configuration-files.md index df56ab9ac7d..a4cc9182427 100644 --- a/docs/ru/operations/configuration-files.md +++ b/docs/ru/operations/configuration-files.md @@ -1,3 +1,9 @@ +--- +toc_priority: 50 +toc_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u0444\u0430\u0439\u043b\u044b" +--- + + # Конфигурационные файлы {#configuration_files} Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`. diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 0a15959d652..74a1d135967 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -1,7 +1,7 @@ --- toc_folder_title: "\u042d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u044f" toc_priority: 41 -toc_title: intro +toc_title: "\u042d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u044f" --- # Эксплуатация {#operations} diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index a1013f5eae5..52d0b5ecc8a 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u041c\u043e\u043d\u0438\u0442\u043e\u0440\u0438\u043d\u0433" +--- + # Мониторинг {#monitoring} Вы можете отслеживать: diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md index 399e80d2011..92533eef0c1 100644 --- a/docs/ru/operations/quotas.md +++ b/docs/ru/operations/quotas.md @@ -1,3 +1,8 @@ +--- +toc_priority: 51 +toc_title: "\u041a\u0432\u043e\u0442\u044b" +--- + # Квоты {#quotas} Квоты позволяют ограничить использование ресурсов за некоторый интервал времени, или просто подсчитывать использование ресурсов. diff --git a/docs/ru/operations/requirements.md b/docs/ru/operations/requirements.md index e1ecafd9f3b..36a7dd30b34 100644 --- a/docs/ru/operations/requirements.md +++ b/docs/ru/operations/requirements.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: "\u0422\u0440\u0435\u0431\u043e\u0432\u0430\u043d\u0438\u044f" +--- + # Требования {#trebovaniia} ## Процессор {#protsessor} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 2ca881d19f0..17966ef0547 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -1,3 +1,8 @@ +--- +toc_priority: 57 +toc_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b\u0020\u0441\u0435\u0440\u0432\u0435\u0440\u0430" +--- + # Конфигурационные параметры сервера {#server-configuration-parameters-reference} ## builtin_dictionaries_reload_interval {#builtin-dictionaries-reload-interval} diff --git a/docs/ru/operations/settings/constraints-on-settings.md b/docs/ru/operations/settings/constraints-on-settings.md index dd324caecc6..b23be22958c 100644 --- a/docs/ru/operations/settings/constraints-on-settings.md +++ b/docs/ru/operations/settings/constraints-on-settings.md @@ -1,3 +1,8 @@ +--- +toc_priority: 62 +toc_title: "\u041e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u0020\u043d\u0430\u0020\u0438\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +--- + # Ограничения на изменение настроек {#constraints-on-settings} Ограничения на изменение настроек могут находиться внутри секции `profiles` файла `user.xml` и запрещают пользователю менять некоторые настройки с помощью запроса `SET`. diff --git a/docs/ru/operations/settings/index.md b/docs/ru/operations/settings/index.md index edc8d2d3014..c24b7053c46 100644 --- a/docs/ru/operations/settings/index.md +++ b/docs/ru/operations/settings/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_folder_title: Settings toc_priority: 55 -toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438" +toc_title: Introduction --- # Настройки {#session-settings-intro} diff --git a/docs/ru/operations/settings/permissions-for-queries.md b/docs/ru/operations/settings/permissions-for-queries.md index cb65e652331..ae896dac77c 100644 --- a/docs/ru/operations/settings/permissions-for-queries.md +++ b/docs/ru/operations/settings/permissions-for-queries.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: "\u0420\u0430\u0437\u0440\u0435\u0448\u0435\u043d\u0438\u044f\u0020\u0434\u043b\u044f\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432" +--- + # Разрешения для запросов {#permissions_for_queries} Запросы в ClickHouse можно разделить на несколько типов: diff --git a/docs/ru/operations/settings/query-complexity.md b/docs/ru/operations/settings/query-complexity.md index aeb5445a054..a62e7523207 100644 --- a/docs/ru/operations/settings/query-complexity.md +++ b/docs/ru/operations/settings/query-complexity.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: "\u041e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u0020\u043d\u0430\u0020\u0441\u043b\u043e\u0436\u043d\u043e\u0441\u0442\u044c\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u0430" +--- + # Ограничения на сложность запроса {#restrictions-on-query-complexity} Ограничения на сложность запроса - часть настроек. diff --git a/docs/ru/operations/settings/settings-profiles.md b/docs/ru/operations/settings/settings-profiles.md index d1e24490120..10feda01850 100644 --- a/docs/ru/operations/settings/settings-profiles.md +++ b/docs/ru/operations/settings/settings-profiles.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u0438\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +--- + # Профили настроек {#settings-profiles} Профиль настроек — это набор настроек, сгруппированных под одним именем. diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md index 7c12780823a..2069922d0ea 100644 --- a/docs/ru/operations/settings/settings-users.md +++ b/docs/ru/operations/settings/settings-users.md @@ -1,3 +1,8 @@ +--- +toc_priority: 63 +toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438\u0020\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435\u0439" +--- + # Настройки пользователей {#nastroiki-polzovatelei} Раздел `users` конфигурационного файла `user.xml` содержит настройки для пользователей. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 8a65f3781c1..3192b523d48 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1,6 +1,6 @@ --- toc_priority: 60 -toc_title: Settings +toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438" --- # Настройки {#settings} diff --git a/docs/ru/operations/system-tables/index.md b/docs/ru/operations/system-tables/index.md index 6fa989d3d0d..93ea1c92068 100644 --- a/docs/ru/operations/system-tables/index.md +++ b/docs/ru/operations/system-tables/index.md @@ -1,6 +1,6 @@ --- toc_priority: 52 -toc_title: Системные таблицы +toc_title: "\u0421\u0438\u0441\u0442\u0435\u043c\u043d\u044b\u0435\u0020\u0442\u0430\u0431\u043b\u0438\u0446\u044b" --- # Системные таблицы {#system-tables} diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md index 27ed93e874e..39f685288d8 100644 --- a/docs/ru/operations/system-tables/query_log.md +++ b/docs/ru/operations/system-tables/query_log.md @@ -33,11 +33,12 @@ ClickHouse не удаляет данные из таблица автомати - `'ExceptionWhileProcessing' = 4` — исключение во время обработки запроса. - `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата начала запроса. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала запроса. +- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала запроса с точностью до микросекунд. - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность выполнения запроса в миллисекундах. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. - `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. - `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество строк в результате запроса `SELECT` или количество строк в запросе `INSERT`. @@ -76,64 +77,67 @@ ClickHouse не удаляет данные из таблица автомати - `quota_key` ([String](../../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse. - `thread_numbers` ([Array(UInt32)](../../sql-reference/data-types/array.md)) — количество потоков, участвующих в обработке запросов. -- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(#system_tables-events +- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(#system_tables-events - `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — метрики, перечисленные в столбце `ProfileEvents.Names`. - `Settings.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1. -- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — Значения настроек, которые перечислены в столбце `Settings.Names`. +- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — значения настроек, которые перечислены в столбце `Settings.Names`. **Пример** ``` sql -SELECT * FROM system.query_log LIMIT 1 FORMAT Vertical; +SELECT * FROM system.query_log LIMIT 1 \G ``` ``` text Row 1: ────── -type: QueryStart -event_date: 2020-05-13 -event_time: 2020-05-13 14:02:28 -query_start_time: 2020-05-13 14:02:28 -query_duration_ms: 0 -read_rows: 0 -read_bytes: 0 -written_rows: 0 -written_bytes: 0 -result_rows: 0 -result_bytes: 0 -memory_usage: 0 -query: SELECT 1 -exception_code: 0 -exception: -stack_trace: -is_initial_query: 1 -user: default -query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -address: ::ffff:127.0.0.1 -port: 57720 -initial_user: default -initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -initial_address: ::ffff:127.0.0.1 -initial_port: 57720 -interface: 1 -os_user: bayonet -client_hostname: clickhouse.ru-central1.internal -client_name: ClickHouse client -client_revision: 54434 -client_version_major: 20 -client_version_minor: 4 -client_version_patch: 1 -http_method: 0 -http_user_agent: -quota_key: -revision: 54434 -thread_ids: [] -ProfileEvents.Names: [] -ProfileEvents.Values: [] -Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage'] -Settings.Values: ['0','random','1','10000000000'] - +type: QueryStart +event_date: 2020-09-11 +event_time: 2020-09-11 10:08:17 +event_time_microseconds: 2020-09-11 10:08:17.063321 +query_start_time: 2020-09-11 10:08:17 +query_start_time_microseconds: 2020-09-11 10:08:17.063321 +query_duration_ms: 0 +read_rows: 0 +read_bytes: 0 +written_rows: 0 +written_bytes: 0 +result_rows: 0 +result_bytes: 0 +memory_usage: 0 +current_database: default +query: INSERT INTO test1 VALUES +exception_code: 0 +exception: +stack_trace: +is_initial_query: 1 +user: default +query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +address: ::ffff:127.0.0.1 +port: 33452 +initial_user: default +initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +initial_address: ::ffff:127.0.0.1 +initial_port: 33452 +interface: 1 +os_user: bharatnc +client_hostname: tower +client_name: ClickHouse +client_revision: 54437 +client_version_major: 20 +client_version_minor: 7 +client_version_patch: 2 +http_method: 0 +http_user_agent: +quota_key: +revision: 54440 +thread_ids: [] +ProfileEvents.Names: [] +ProfileEvents.Values: [] +Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage','allow_introspection_functions'] +Settings.Values: ['0','random','1','10000000000','1'] ``` + **Смотрите также** - [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — в этой таблице содержится информация о цепочке каждого выполненного запроса. diff --git a/docs/ru/operations/system-tables/query_thread_log.md b/docs/ru/operations/system-tables/query_thread_log.md index 11571f594d5..052baf98035 100644 --- a/docs/ru/operations/system-tables/query_thread_log.md +++ b/docs/ru/operations/system-tables/query_thread_log.md @@ -15,6 +15,7 @@ ClickHouse не удаляет данные из таблицы автомати - `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком. +- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком с точностью до микросекунд. - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах. @@ -24,7 +25,7 @@ ClickHouse не удаляет данные из таблицы автомати - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. - `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — разница между выделенной и освобождённой памятью в контексте потока. - `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — максимальная разница между выделенной и освобождённой памятью в контексте потока. -- `thread_name` ([String](../../sql-reference/data-types/string.md)) — Имя потока. +- `thread_name` ([String](../../sql-reference/data-types/string.md)) — имя потока. - `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы). - `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы) главного потока. - `query` ([String](../../sql-reference/data-types/string.md)) — текст запроса. @@ -56,56 +57,57 @@ ClickHouse не удаляет данные из таблицы автомати - `http_user_agent` ([String](../../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`. - `quota_key` ([String](../../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse. -- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events). +- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events). - `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. **Пример** ``` sql - SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical + SELECT * FROM system.query_thread_log LIMIT 1 \G ``` ``` text Row 1: ────── -event_date: 2020-05-13 -event_time: 2020-05-13 14:02:28 -query_start_time: 2020-05-13 14:02:28 -query_duration_ms: 0 -read_rows: 1 -read_bytes: 1 -written_rows: 0 -written_bytes: 0 -memory_usage: 0 -peak_memory_usage: 0 -thread_name: QueryPipelineEx -thread_id: 28952 -master_thread_id: 28924 -query: SELECT 1 -is_initial_query: 1 -user: default -query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -address: ::ffff:127.0.0.1 -port: 57720 -initial_user: default -initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -initial_address: ::ffff:127.0.0.1 -initial_port: 57720 -interface: 1 -os_user: bayonet -client_hostname: clickhouse.ru-central1.internal -client_name: ClickHouse client -client_revision: 54434 -client_version_major: 20 -client_version_minor: 4 -client_version_patch: 1 -http_method: 0 -http_user_agent: -quota_key: -revision: 54434 -ProfileEvents.Names: ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds'] -ProfileEvents.Values: [1,97,81,5,81] -... +event_date: 2020-09-11 +event_time: 2020-09-11 10:08:17 +event_time_microseconds: 2020-09-11 10:08:17.134042 +query_start_time: 2020-09-11 10:08:17 +query_start_time_microseconds: 2020-09-11 10:08:17.063150 +query_duration_ms: 70 +read_rows: 0 +read_bytes: 0 +written_rows: 1 +written_bytes: 12 +memory_usage: 4300844 +peak_memory_usage: 4300844 +thread_name: TCPHandler +thread_id: 638133 +master_thread_id: 638133 +query: INSERT INTO test1 VALUES +is_initial_query: 1 +user: default +query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +address: ::ffff:127.0.0.1 +port: 33452 +initial_user: default +initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +initial_address: ::ffff:127.0.0.1 +initial_port: 33452 +interface: 1 +os_user: bharatnc +client_hostname: tower +client_name: ClickHouse +client_revision: 54437 +client_version_major: 20 +client_version_minor: 7 +client_version_patch: 2 +http_method: 0 +http_user_agent: +quota_key: +revision: 54440 +ProfileEvents.Names: ['Query','InsertQuery','FileOpen','WriteBufferFromFileDescriptorWrite','WriteBufferFromFileDescriptorWriteBytes','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','FunctionExecute','CreatedWriteBufferOrdinary','DiskWriteElapsedMicroseconds','NetworkReceiveElapsedMicroseconds','NetworkSendElapsedMicroseconds','InsertedRows','InsertedBytes','SelectedRows','SelectedBytes','MergeTreeDataWriterRows','MergeTreeDataWriterUncompressedBytes','MergeTreeDataWriterCompressedBytes','MergeTreeDataWriterBlocks','MergeTreeDataWriterBlocksAlreadySorted','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SoftPageFaults','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSReadChars','OSWriteChars'] +ProfileEvents.Values: [1,1,11,11,591,148,3,71,29,6533808,1,11,72,18,47,1,12,1,12,1,12,189,1,1,10,2,70853,2748,49,2747,45056,422,1520] ``` **Смотрите также** diff --git a/docs/ru/operations/system-tables/text_log.md b/docs/ru/operations/system-tables/text_log.md index 01e34c914e6..141c3680c07 100644 --- a/docs/ru/operations/system-tables/text_log.md +++ b/docs/ru/operations/system-tables/text_log.md @@ -4,12 +4,13 @@ Столбцы: -- `event_date` (Date) — Дата создания записи. -- `event_time` (DateTime) — Время создания записи. -- `microseconds` (UInt32) — Время создания записи в микросекундах. -- `thread_name` (String) — Название потока, из которого была сделана запись. -- `thread_id` (UInt64) — Идентификатор потока ОС. -- `level` (Enum8) — Уровень логирования записи. Возможные значения: +- `event_date` (Date) — дата создания записи. +- `event_time` (DateTime) — время создания записи. +- `event_time_microseconds` (DateTime) — время создания записи с точностью до микросекунд. +- `microseconds` (UInt32) — время создания записи в микросекундах. +- `thread_name` (String) — название потока, из которого была сделана запись. +- `thread_id` (UInt64) — идентификатор потока ОС. +- `level` (Enum8) — уровень логирования записи. Возможные значения: - `1` или `'Fatal'`. - `2` или `'Critical'`. - `3` или `'Error'`. @@ -18,11 +19,35 @@ - `6` или `'Information'`. - `7` или `'Debug'`. - `8` или `'Trace'`. -- `query_id` (String) — Идентификатор запроса. -- `logger_name` (LowCardinality(String)) — Название логгера (`DDLWorker`). -- `message` (String) — Само тело записи. -- `revision` (UInt32) — Ревизия ClickHouse. -- `source_file` (LowCardinality(String)) — Исходный файл, из которого была сделана запись. -- `source_line` (UInt64) — Исходная строка, из которой была сделана запись. +- `query_id` (String) — идентификатор запроса. +- `logger_name` (LowCardinality(String)) — название логгера (`DDLWorker`). +- `message` (String) — само тело записи. +- `revision` (UInt32) — ревизия ClickHouse. +- `source_file` (LowCardinality(String)) — исходный файл, из которого была сделана запись. +- `source_line` (UInt64) — исходная строка, из которой была сделана запись. + +**Пример** + +``` sql +SELECT * FROM system.text_log LIMIT 1 \G +``` + +``` text +Row 1: +────── +event_date: 2020-09-10 +event_time: 2020-09-10 11:23:07 +event_time_microseconds: 2020-09-10 11:23:07.871397 +microseconds: 871397 +thread_name: clickhouse-serv +thread_id: 564917 +level: Information +query_id: +logger_name: DNSCacheUpdater +message: Update period 15 seconds +revision: 54440 +source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start() +source_line: 45 +``` [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/text_log) diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 8bab8ff646c..3f0a16199d5 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -6,26 +6,28 @@ ClickHouse создает эту таблицу когда утсановлен Для анализа stack traces, используйте функции интроспекции `addressToLine`, `addressToSymbol` и `demangle`. -Колонки: +Столбцы: -- `event_date`([Date](../../sql-reference/data-types/date.md)) — Дата в момент снятия экземпляра стэка адресов вызова. +- `event_date`([Date](../../sql-reference/data-types/date.md)) — дата в момент снятия экземпляра стэка адресов вызова. -- `event_time`([DateTime](../../sql-reference/data-types/datetime.md)) — Дата и время в момент снятия экземпляра стэка адресов вызова. +- `event_time`([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время в момент снятия экземпляра стэка адресов вызова. + +- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время в момент снятия экземпляра стэка адресов вызова с точностью до микросекунд. - `revision`([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия сборки сервера ClickHouse. Во время соединения с сервером через `clickhouse-client`, вы видите строку похожую на `Connected to ClickHouse server version 19.18.1 revision 54429.`. Это поле содержит номер после `revision`, но не содержит строку после `version`. -- `timer_type`([Enum8](../../sql-reference/data-types/enum.md)) — Тип таймера: +- `timer_type`([Enum8](../../sql-reference/data-types/enum.md)) — тип таймера: - `Real` означает wall-clock время. - `CPU` означает относительное CPU время. -- `thread_number`([UInt32](../../sql-reference/data-types/int-uint.md)) — Идентификатор треда. +- `thread_number`([UInt32](../../sql-reference/data-types/int-uint.md)) — идентификатор треда. -- `query_id`([String](../../sql-reference/data-types/string.md)) — Идентификатор запроса который может быть использован для получения деталей о запросе из таблицы [query_log](query_log.md#system_tables-query_log) system table. +- `query_id`([String](../../sql-reference/data-types/string.md)) — идентификатор запроса который может быть использован для получения деталей о запросе из таблицы [query_log](query_log.md#system_tables-query_log) system table. -- `trace`([Array(UInt64)](../../sql-reference/data-types/array.md)) — Трассировка стека адресов вызова в момент семплирования. Каждый элемент массива это адрес виртуальной памяти внутри процесса сервера ClickHouse. +- `trace`([Array(UInt64)](../../sql-reference/data-types/array.md)) — трассировка стека адресов вызова в момент семплирования. Каждый элемент массива — это адрес виртуальной памяти внутри процесса сервера ClickHouse. **Пример** @@ -36,13 +38,16 @@ SELECT * FROM system.trace_log LIMIT 1 \G ``` text Row 1: ────── -event_date: 2019-11-15 -event_time: 2019-11-15 15:09:38 -revision: 54428 -timer_type: Real -thread_number: 48 -query_id: acc4d61f-5bd1-4a3e-bc91-2180be37c915 -trace: [94222141367858,94222152240175,94222152325351,94222152329944,94222152330796,94222151449980,94222144088167,94222151682763,94222144088167,94222151682763,94222144088167,94222144058283,94222144059248,94222091840750,94222091842302,94222091831228,94222189631488,140509950166747,140509942945935] +event_date: 2020-09-10 +event_time: 2020-09-10 11:23:09 +event_time_microseconds: 2020-09-10 11:23:09.872924 +timestamp_ns: 1599762189872924510 +revision: 54440 +trace_type: Memory +thread_id: 564963 +query_id: +trace: [371912858,371912789,371798468,371799717,371801313,371790250,624462773,566365041,566440261,566445834,566460071,566459914,566459842,566459580,566459469,566459389,566459341,566455774,371993941,371988245,372158848,372187428,372187309,372187093,372185478,140222123165193,140222122205443] +size: 5244400 ``` [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/trace_log) diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md index f076fab1311..40035309c03 100644 --- a/docs/ru/operations/tips.md +++ b/docs/ru/operations/tips.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: "\u0421\u043e\u0432\u0435\u0442\u044b\u0020\u043f\u043e\u0020\u044d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u0438" +--- + # Советы по эксплуатации {#sovety-po-ekspluatatsii} ## CPU Scaling Governor {#cpu-scaling-governor} diff --git a/docs/ru/operations/troubleshooting.md b/docs/ru/operations/troubleshooting.md index cb86dfef5e8..3df2a1dd46c 100644 --- a/docs/ru/operations/troubleshooting.md +++ b/docs/ru/operations/troubleshooting.md @@ -1,3 +1,8 @@ +--- +toc_priority: 46 +toc_title: "\u0423\u0441\u0442\u0440\u0430\u043d\u0435\u043d\u0438\u0435\u0020\u043d\u0435\u0438\u0441\u043f\u0440\u0430\u0432\u043d\u043e\u0441\u0442\u0435\u0439" +--- + # Устранение неисправностей {#ustranenie-neispravnostei} - [Установка дистрибутива](#troubleshooting-installation-errors) diff --git a/docs/ru/operations/update.md b/docs/ru/operations/update.md index f1998864f40..c74b28b3fd7 100644 --- a/docs/ru/operations/update.md +++ b/docs/ru/operations/update.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: "\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Обновление ClickHouse {#obnovlenie-clickhouse} Если ClickHouse установлен с помощью deb-пакетов, выполните следующие команды на сервере: diff --git a/docs/ru/operations/utilities/clickhouse-copier.md b/docs/ru/operations/utilities/clickhouse-copier.md index b43f5ccaf7a..64e3c1eee12 100644 --- a/docs/ru/operations/utilities/clickhouse-copier.md +++ b/docs/ru/operations/utilities/clickhouse-copier.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: clickhouse-copier +--- + # clickhouse-copier {#clickhouse-copier} Копирует данные из таблиц одного кластера в таблицы другого (или этого же) кластера. diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index 7dfa9587686..962b63e2b70 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: clickhouse-local +--- + # clickhouse-local {#clickhouse-local} Принимает на вход данные, которые можно представить в табличном виде и выполняет над ними операции, заданные на [языке запросов](../../operations/utilities/clickhouse-local.md) ClickHouse. diff --git a/docs/ru/operations/utilities/index.md b/docs/ru/operations/utilities/index.md index 91ee649ee3a..5b55ebd798d 100644 --- a/docs/ru/operations/utilities/index.md +++ b/docs/ru/operations/utilities/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: "\u0423\u0442\u0438\u043b\u0438\u0442\u044b" +toc_priority: 56 +toc_title: "\u041e\u0431\u0437\u043e\u0440" +--- + # Утилиты ClickHouse {#utility-clickhouse} - [clickhouse-local](clickhouse-local.md) diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index ca65f6ac615..ef014906423 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -1,3 +1,9 @@ +--- +toc_priority: 37 +toc_title: "\u041a\u043e\u043c\u0431\u0438\u043d\u0430\u0442\u043e\u0440\u044b\u0020\u0430\u0433\u0440\u0435\u0433\u0430\u0442\u043d\u044b\u0445\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0439" +--- + + # Комбинаторы агрегатных функций {#aggregate_functions_combinators} К имени агрегатной функции может быть приписан некоторый суффикс. При этом, работа агрегатной функции некоторым образом модифицируется. diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 7cc0bdef581..f20acaa45c3 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: "\u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0430\u0433\u0440\u0435\u0433\u0430\u0442\u043d\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Параметрические агрегатные функции {#aggregate_functions_parametric} Некоторые агрегатные функции могут принимать не только столбцы-аргументы (по которым производится свёртка), но и набор параметров - констант для инициализации. Синтаксис - две пары круглых скобок вместо одной. Первая - для параметров, вторая - для аргументов. diff --git a/docs/ru/sql-reference/data-types/boolean.md b/docs/ru/sql-reference/data-types/boolean.md index f868ebf6d14..bb0cd50c739 100644 --- a/docs/ru/sql-reference/data-types/boolean.md +++ b/docs/ru/sql-reference/data-types/boolean.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: "\u0411\u0443\u043b\u0435\u0432\u044b\u0020\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f" +--- + # Булевы значения {#bulevy-znacheniia} Отдельного типа для булевых значений нет. Для них используется тип UInt8, в котором используются только значения 0 и 1. diff --git a/docs/ru/sql-reference/data-types/date.md b/docs/ru/sql-reference/data-types/date.md index 97f58ec30f6..9bcae2c1d72 100644 --- a/docs/ru/sql-reference/data-types/date.md +++ b/docs/ru/sql-reference/data-types/date.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: Date +--- + # Date {#data-type-date} Дата. Хранится в двух байтах в виде (беззнакового) числа дней, прошедших от 1970-01-01. Позволяет хранить значения от чуть больше, чем начала unix-эпохи до верхнего порога, определяющегося константой на этапе компиляции (сейчас - до 2106 года, последний полностью поддерживаемый год - 2105). diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index d60e81c74d5..87c5da68f35 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -1,3 +1,8 @@ +--- +toc_priority: 48 +toc_title: DateTime +--- + # DateTime {#data_type-datetime} Позволяет хранить момент времени, который может быть представлен как календарная дата и время. diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md index bc9394276ee..0a602e44636 100644 --- a/docs/ru/sql-reference/data-types/datetime64.md +++ b/docs/ru/sql-reference/data-types/datetime64.md @@ -1,3 +1,8 @@ +--- +toc_priority: 49 +toc_title: DateTime64 +--- + # DateTime64 {#data_type-datetime64} Позволяет хранить момент времени, который может быть представлен как календарная дата и время, с заданной суб-секундной точностью. diff --git a/docs/ru/sql-reference/data-types/decimal.md b/docs/ru/sql-reference/data-types/decimal.md index 29192cb8819..bdcd3c767b9 100644 --- a/docs/ru/sql-reference/data-types/decimal.md +++ b/docs/ru/sql-reference/data-types/decimal.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: Decimal +--- + # Decimal(P, S), Decimal32(S), Decimal64(S), Decimal128(S), Decimal256(S) {#decimalp-s-decimal32s-decimal64s-decimal128s} Знаковые дробные числа с сохранением точности операций сложения, умножения и вычитания. Для деления осуществляется отбрасывание (не округление) знаков, не попадающих в младший десятичный разряд. diff --git a/docs/ru/sql-reference/data-types/domains/index.md b/docs/ru/sql-reference/data-types/domains/index.md index fe5c7ab7349..4449469b1bc 100644 --- a/docs/ru/sql-reference/data-types/domains/index.md +++ b/docs/ru/sql-reference/data-types/domains/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: Домены -toc_title_title: Обзор +toc_folder_title: "\u0414\u043e\u043c\u0435\u043d\u044b" +toc_title_title: "\u041e\u0431\u0437\u043e\u0440" toc_priority: 56 --- diff --git a/docs/ru/sql-reference/data-types/domains/ipv4.md b/docs/ru/sql-reference/data-types/domains/ipv4.md index 68b67bcca60..57d6f12ab17 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv4.md +++ b/docs/ru/sql-reference/data-types/domains/ipv4.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: IPv4 +--- + ## IPv4 {#ipv4} `IPv4` — это домен, базирующийся на типе данных `UInt32` предназначенный для хранения адресов IPv4. Он обеспечивает компактное хранение данных с удобным для человека форматом ввода-вывода, и явно отображаемым типом данных в структуре таблицы. diff --git a/docs/ru/sql-reference/data-types/domains/ipv6.md b/docs/ru/sql-reference/data-types/domains/ipv6.md index c88ee74adea..04c5fd0d491 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv6.md +++ b/docs/ru/sql-reference/data-types/domains/ipv6.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: IPv6 +--- + ## IPv6 {#ipv6} `IPv6` — это домен, базирующийся на типе данных `FixedString(16)`, предназначенный для хранения адресов IPv6. Он обеспечивает компактное хранение данных с удобным для человека форматом ввода-вывода, и явно отображаемым типом данных в структуре таблицы. diff --git a/docs/ru/sql-reference/data-types/enum.md b/docs/ru/sql-reference/data-types/enum.md index 792c82f7410..b86d15c19a8 100644 --- a/docs/ru/sql-reference/data-types/enum.md +++ b/docs/ru/sql-reference/data-types/enum.md @@ -1,3 +1,8 @@ +--- +toc_priority: 50 +toc_title: Enum +--- + # Enum {#enum} Перечисляемый тип данных, содержащий именованные значения. diff --git a/docs/ru/sql-reference/data-types/fixedstring.md b/docs/ru/sql-reference/data-types/fixedstring.md index 4a26b2f76af..21115418e30 100644 --- a/docs/ru/sql-reference/data-types/fixedstring.md +++ b/docs/ru/sql-reference/data-types/fixedstring.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: FixedString(N) +--- + # FixedString {#fixedstring} Строка фиксированной длины `N` байт (не символов, не кодовых точек). diff --git a/docs/ru/sql-reference/data-types/float.md b/docs/ru/sql-reference/data-types/float.md index 91d4b655e2a..f2e85f35041 100644 --- a/docs/ru/sql-reference/data-types/float.md +++ b/docs/ru/sql-reference/data-types/float.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: Float32, Float64 +--- + # Float32, Float64 {#float32-float64} [Числа с плавающей запятой](https://en.wikipedia.org/wiki/IEEE_754). diff --git a/docs/ru/sql-reference/data-types/int-uint.md b/docs/ru/sql-reference/data-types/int-uint.md index 3a33c95e4c3..c45c639aace 100644 --- a/docs/ru/sql-reference/data-types/int-uint.md +++ b/docs/ru/sql-reference/data-types/int-uint.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 +--- + # UInt8, UInt16, UInt32, UInt64, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 {#uint8-uint16-uint32-uint64-int8-int16-int32-int64} Целые числа фиксированной длины, без знака или со знаком. diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/index.md b/docs/ru/sql-reference/data-types/nested-data-structures/index.md index 73d9019c96a..d53cabc6652 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/index.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0412\u043b\u043e\u0436\u0435\u043d\u043d\u044b\u0435\u0020\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_hidden: true +toc_priority: 54 +toc_title: hidden +--- + # Вложенные структуры данных {#vlozhennye-struktury-dannykh} [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/) diff --git a/docs/ru/sql-reference/data-types/special-data-types/expression.md b/docs/ru/sql-reference/data-types/special-data-types/expression.md index 1f4b960fb10..718fcc886a6 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/expression.md +++ b/docs/ru/sql-reference/data-types/special-data-types/expression.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: Expression +--- + # Expression {#expression} Используется для представления лямбда-выражений в функциях высшего порядка. diff --git a/docs/ru/sql-reference/data-types/special-data-types/index.md b/docs/ru/sql-reference/data-types/special-data-types/index.md index d5aff1501db..29c057472ea 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/index.md +++ b/docs/ru/sql-reference/data-types/special-data-types/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0421\u043b\u0443\u0436\u0435\u0431\u043d\u044b\u0435\u0020\u0442\u0438\u043f\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_hidden: true +toc_priority: 55 +toc_title: hidden +--- + # Служебные типы данных {#sluzhebnye-tipy-dannykh} Значения служебных типов данных не могут сохраняться в таблицу и выводиться в качестве результата, а возникают как промежуточный результат выполнения запроса. diff --git a/docs/ru/sql-reference/data-types/special-data-types/interval.md b/docs/ru/sql-reference/data-types/special-data-types/interval.md index a77d05ab8be..31240b49c97 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/interval.md +++ b/docs/ru/sql-reference/data-types/special-data-types/interval.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: Interval +--- + # Interval {#data-type-interval} Семейство типов данных, представляющих интервалы дат и времени. Оператор [INTERVAL](../../../sql-reference/data-types/special-data-types/interval.md#operator-interval) возвращает значения этих типов. diff --git a/docs/ru/sql-reference/data-types/special-data-types/nothing.md b/docs/ru/sql-reference/data-types/special-data-types/nothing.md index 9644f102522..c6a9cb868d8 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/nothing.md +++ b/docs/ru/sql-reference/data-types/special-data-types/nothing.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: Nothing +--- + # Nothing {#nothing} Этот тип данных предназначен только для того, чтобы представлять [NULL](../../../sql-reference/data-types/special-data-types/nothing.md), т.е. отсутствие значения. diff --git a/docs/ru/sql-reference/data-types/special-data-types/set.md b/docs/ru/sql-reference/data-types/special-data-types/set.md index 9a99ed0ca9b..4c2f4ed2c66 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/set.md +++ b/docs/ru/sql-reference/data-types/special-data-types/set.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: Set +--- + # Set {#set} Используется для представления правой части выражения IN. diff --git a/docs/ru/sql-reference/data-types/string.md b/docs/ru/sql-reference/data-types/string.md index 4669d154df2..798caec4d62 100644 --- a/docs/ru/sql-reference/data-types/string.md +++ b/docs/ru/sql-reference/data-types/string.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: String +--- + # String {#string} Строки произвольной длины. Длина не ограничена. Значение может содержать произвольный набор байт, включая нулевые байты. diff --git a/docs/ru/sql-reference/data-types/uuid.md b/docs/ru/sql-reference/data-types/uuid.md index 9d667a50526..b780190f6f4 100644 --- a/docs/ru/sql-reference/data-types/uuid.md +++ b/docs/ru/sql-reference/data-types/uuid.md @@ -1,3 +1,9 @@ +--- +toc_priority: 46 +toc_title: UUID +--- + + # UUID {#uuid-data-type} Универсальный уникальный идентификатор (UUID) - это 16-байтовое число, используемое для идентификации записей. Подробнее про UUID читайте на [Википедии](https://en.wikipedia.org/wiki/Universally_unique_identifier). diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 0869b409b0b..350e391dbed 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u0418\u0435\u0440\u0430\u0440\u0445\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +--- + # Иерархические словари {#ierarkhicheskie-slovari} ClickHouse поддерживает иерархические словари с [числовыми ключом](external-dicts-dict-structure.md#ext_dict-numeric-key). diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 61275cc6652..979f1f2e5b9 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: "\u0425\u0440\u0430\u043d\u0435\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439\u0020\u0432\u0020\u043f\u0430\u043c\u044f\u0442\u0438" +--- + # Хранение словарей в памяти {#dicts-external-dicts-dict-layout} Словари можно размещать в памяти множеством способов. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 2f287795296..4dbf4be9f96 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: "\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439" +--- + # Обновление словарей {#obnovlenie-slovarei} ClickHouse периодически обновляет словари. Интервал обновления для полностью загружаемых словарей и интервал инвалидации для кэшируемых словарей определяется в теге `` в секундах. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 61d98f37cfd..0015edfdf72 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: "\u0418\u0441\u0442\u043e\u0447\u043d\u0438\u043a\u0438\u0020\u0432\u043d\u0435\u0448\u043d\u0438\u0445\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439" +--- + # Источники внешних словарей {#dicts-external-dicts-dict-sources} Внешний словарь можно подключить из множества источников. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 4c3b4eb22e4..bf87ce61b9e 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: "\u041a\u043b\u044e\u0447\u0020\u0438\u0020\u043f\u043e\u043b\u044f\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f" +--- + # Ключ и поля словаря {#kliuch-i-polia-slovaria} Секция `` описывает ключ словаря и поля, доступные для запросов. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 4ecc9b6c093..ff18f906926 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u0020\u0432\u043d\u0435\u0448\u043d\u0435\u0433\u043e\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f" +--- + # Настройка внешнего словаря {#dicts-external-dicts-dict} XML-конфигурация словаря имеет следующую структуру: diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 734de8cffdf..c18af68c15e 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -1,3 +1,9 @@ +--- +toc_priority: 39 +toc_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +--- + + # Внешние словари {#dicts-external-dicts} Существует возможность подключать собственные словари из различных источников данных. Источником данных для словаря может быть локальный текстовый/исполняемый файл, HTTP(s) ресурс или другая СУБД. Подробнее смотрите в разделе «[Источники внешних словарей](external-dicts-dict-sources.md)». diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md index 4098ac38060..b448858b1fa 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: External Dictionaries +toc_folder_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" toc_priority: 37 --- diff --git a/docs/ru/sql-reference/dictionaries/index.md b/docs/ru/sql-reference/dictionaries/index.md index 8625a29b1e1..5a4119b4dd5 100644 --- a/docs/ru/sql-reference/dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: "\u0421\u043b\u043e\u0432\u0430\u0440\u0438" +toc_priority: 35 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + # Словари {#slovari} Словарь — это отображение (`ключ -> атрибуты`), которое удобно использовать для различного вида справочников. diff --git a/docs/ru/sql-reference/dictionaries/internal-dicts.md b/docs/ru/sql-reference/dictionaries/internal-dicts.md index 0e8e7c82a6e..d8103efa6ae 100644 --- a/docs/ru/sql-reference/dictionaries/internal-dicts.md +++ b/docs/ru/sql-reference/dictionaries/internal-dicts.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: "\u0412\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u044b\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +--- + # Встроенные словари {#internal_dicts} ClickHouse содержит встроенную возможность работы с геобазой. diff --git a/docs/ru/sql-reference/distributed-ddl.md b/docs/ru/sql-reference/distributed-ddl.md index a06a931680a..275709320f6 100644 --- a/docs/ru/sql-reference/distributed-ddl.md +++ b/docs/ru/sql-reference/distributed-ddl.md @@ -1,6 +1,6 @@ --- toc_priority: 32 -toc_title: Распределенные DDL запросы +toc_title: "\u0420\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0435\u0020\u0044\u0044\u004c\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u044b\u000a" --- # Распределенные DDL запросы (секция ON CLUSTER) {#raspredelennye-ddl-zaprosy-sektsiia-on-cluster} diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index 8513737f025..16c3e8fd8f0 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: "\u0410\u0440\u0438\u0444\u043c\u0435\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Арифметические функции {#arifmeticheskie-funktsii} Для всех арифметических функций, тип результата вычисляется, как минимальный числовой тип, который может вместить результат, если такой тип есть. Минимум берётся одновременно по числу бит, знаковости и «плавучести». Если бит не хватает, то берётся тип максимальной битности. diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index 91c0443c85d..62181eebb4b 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: "\u041c\u0430\u0441\u0441\u0438\u0432\u044b" +--- + # Массивы {#functions-for-working-with-arrays} ## empty {#function-empty} diff --git a/docs/ru/sql-reference/functions/array-join.md b/docs/ru/sql-reference/functions/array-join.md index afbc7bfe3e8..2ed3d25fa92 100644 --- a/docs/ru/sql-reference/functions/array-join.md +++ b/docs/ru/sql-reference/functions/array-join.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u044f\u0020\u0041\u0072\u0072\u0061\u0079\u004a\u006f\u0069\u006e" +--- + # Функция ArrayJoin {#functions_arrayjoin} Это совсем необычная функция. diff --git a/docs/ru/sql-reference/functions/bit-functions.md b/docs/ru/sql-reference/functions/bit-functions.md index 9d78dd861e9..8c7808437a5 100644 --- a/docs/ru/sql-reference/functions/bit-functions.md +++ b/docs/ru/sql-reference/functions/bit-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 48 +toc_title: "\u0411\u0438\u0442\u043e\u0432\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Битовые функции {#bitovye-funktsii} Битовые функции работают для любой пары типов из UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64. diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index aa154831b48..b21ddea94e4 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -1,4 +1,9 @@ -# Функции для битмапов {#bitmap-functions} +--- +toc_priority: 49 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0431\u0438\u0442\u043c\u0430\u043f\u043e\u0432" +--- + +# Функции для битовых масок {#bitmap-functions} ## bitmapBuild {#bitmap_functions-bitmapbuild} diff --git a/docs/ru/sql-reference/functions/comparison-functions.md b/docs/ru/sql-reference/functions/comparison-functions.md index 9b921d6fe55..a98c97ec96c 100644 --- a/docs/ru/sql-reference/functions/comparison-functions.md +++ b/docs/ru/sql-reference/functions/comparison-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044f" +--- + # Функции сравнения {#funktsii-sravneniia} Функции сравнения возвращают всегда 0 или 1 (UInt8). diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 7efb6f7bfc5..83268b68959 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: "\u0423\u0441\u043b\u043e\u0432\u043d\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Условные функции {#uslovnye-funktsii} ## if {#if} diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index aa9fdee478d..d24de2faae1 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0434\u0430\u0442\u0430\u043c\u0438\u0020\u0438\u0020\u0432\u0440\u0435\u043c\u0435\u043d\u0435\u043c" +--- + # Функции для работы с датами и временем {#funktsii-dlia-raboty-s-datami-i-vremenem} Поддержка часовых поясов diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index b8ea481364d..6f1c2aad6cb 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 52 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043a\u043e\u0434\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +--- + # Функции кодирования {#funktsii-kodirovaniia} ## char {#char} diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index 792afd1775d..6054ed141d4 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0432\u043d\u0435\u0448\u043d\u0438\u043c\u0438\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f\u043c\u0438" +--- + # Функции для работы с внешними словарями {#ext_dict_functions} Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). diff --git a/docs/ru/sql-reference/functions/functions-for-nulls.md b/docs/ru/sql-reference/functions/functions-for-nulls.md index 5fbfb4bd27b..17da1ea9194 100644 --- a/docs/ru/sql-reference/functions/functions-for-nulls.md +++ b/docs/ru/sql-reference/functions/functions-for-nulls.md @@ -1,3 +1,8 @@ +--- +toc_priority: 63 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u004e\u0075\u006c\u006c\u0061\u0062\u006c\u0065\u002d\u0430\u0440\u0433\u0443\u043c\u0435\u043d\u0442\u0430\u043c\u0438" +--- + # Функции для работы с Nullable-аргументами {#funktsii-dlia-raboty-s-nullable-argumentami} ## isNull {#isnull} diff --git a/docs/ru/sql-reference/functions/geo/coordinates.md b/docs/ru/sql-reference/functions/geo/coordinates.md index 9d65139e69e..1931a9b932f 100644 --- a/docs/ru/sql-reference/functions/geo/coordinates.md +++ b/docs/ru/sql-reference/functions/geo/coordinates.md @@ -1,5 +1,6 @@ --- -toc_title: Функции для работы с географическими координатами +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0433\u0435\u043e\u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c\u0438\u0020\u043a\u043e\u043e\u0440\u0434\u0438\u043d\u0430\u0442\u0430\u043c\u0438" +toc_priority: 62 --- # Функции для работы с географическими координатами {#geographical-coordinates} diff --git a/docs/ru/sql-reference/functions/geo/geohash.md b/docs/ru/sql-reference/functions/geo/geohash.md index 7273d58b1d9..38c64f11b10 100644 --- a/docs/ru/sql-reference/functions/geo/geohash.md +++ b/docs/ru/sql-reference/functions/geo/geohash.md @@ -1,5 +1,5 @@ --- -toc_title: Geohash +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0441\u0438\u0441\u0442\u0435\u043c\u043e\u0439\u0020\u0047\u0065\u006f\u0068\u0061\u0073\u0068" --- # Функции для работы с системой Geohash {#geohash} diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index d9a5e72721b..69d06b5dfa6 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -1,5 +1,5 @@ --- -toc_title: Индексы H3 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0438\u043d\u0434\u0435\u043a\u0441\u0430\u043c\u0438\u0020\u0048\u0033" --- # Функции для работы с индексами H3 {#h3index} diff --git a/docs/ru/sql-reference/functions/geo/index.md b/docs/ru/sql-reference/functions/geo/index.md index bfea32a245b..cedaafaa31d 100644 --- a/docs/ru/sql-reference/functions/geo/index.md +++ b/docs/ru/sql-reference/functions/geo/index.md @@ -1,6 +1,6 @@ --- toc_priority: 62 -toc_folder_title: Гео-данные +toc_folder_title: "\u0413\u0435\u043e\u002d\u0434\u0430\u043d\u043d\u044b\u0435" toc_title: hidden --- diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 201fc934cea..92fc69227f4 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 50 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0445\u044d\u0448\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +--- + # Функции хэширования {#funktsii-kheshirovaniia} Функции хэширования могут использоваться для детерминированного псевдослучайного разбрасывания элементов. diff --git a/docs/ru/sql-reference/functions/in-functions.md b/docs/ru/sql-reference/functions/in-functions.md index 679fcbccc21..e137187a36b 100644 --- a/docs/ru/sql-reference/functions/in-functions.md +++ b/docs/ru/sql-reference/functions/in-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438\u0020\u043e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u0430\u0020\u0049\u004e" +--- + # Функции для реализации оператора IN {#funktsii-dlia-realizatsii-operatora-in} ## in, notIn, globalIn, globalNotIn {#in-functions} diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index 9c1c0c5ca9d..25d3b6de067 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438" +toc_priority: 32 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + # Функции {#funktsii} Функции бывают как минимум\* двух видов - обычные функции (называются просто, функциями) и агрегатные функции. Это совершенно разные вещи. Обычные функции работают так, как будто применяются к каждой строке по отдельности (для каждой строки, результат вычисления функции не зависит от других строк). Агрегатные функции аккумулируют множество значений из разных строк (то есть, зависят от целого множества строк). diff --git a/docs/ru/sql-reference/functions/introspection.md b/docs/ru/sql-reference/functions/introspection.md index c09efd35093..9f4f2ebd1e9 100644 --- a/docs/ru/sql-reference/functions/introspection.md +++ b/docs/ru/sql-reference/functions/introspection.md @@ -1,3 +1,8 @@ +--- +toc_priority: 65 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0438\u043d\u0442\u0440\u043e\u0441\u043f\u0435\u043a\u0446\u0438\u0438" +--- + # Функции интроспекции {#introspection-functions} Функции из этого раздела могут использоваться для интроспекции [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) и [DWARF](https://en.wikipedia.org/wiki/DWARF) в целях профилирования запросов. diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index a9a0a7f919a..724fb97c0d5 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 55 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0049\u0050\u002d\u0430\u0434\u0440\u0435\u0441\u0430\u043c\u0438" +--- + # Функции для работы с IP-адресами {#funktsii-dlia-raboty-s-ip-adresami} ## IPv4NumToString(num) {#ipv4numtostringnum} diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md index 752b70b7c5f..69b8f8f98f5 100644 --- a/docs/ru/sql-reference/functions/json-functions.md +++ b/docs/ru/sql-reference/functions/json-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 56 +toc_title: JSON +--- + # Функции для работы с JSON {#funktsii-dlia-raboty-s-json} В Яндекс.Метрике пользователями передаётся JSON в качестве параметров визитов. Для работы с таким JSON-ом, реализованы некоторые функции. (Хотя в большинстве случаев, JSON-ы дополнительно обрабатываются заранее, и полученные значения кладутся в отдельные столбцы в уже обработанном виде.) Все эти функции исходят из сильных допущений о том, каким может быть JSON, и при этом стараются почти ничего не делать. diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index de2a3d51729..9b1ee6a66a7 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: "\u041b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Логические функции {#logicheskie-funktsii} Логические функции принимают любые числовые типы, а возвращают число типа UInt8, равное 0 или 1. diff --git a/docs/ru/sql-reference/functions/machine-learning-functions.md b/docs/ru/sql-reference/functions/machine-learning-functions.md index e9b0e8c9bc9..2ffdfd05613 100644 --- a/docs/ru/sql-reference/functions/machine-learning-functions.md +++ b/docs/ru/sql-reference/functions/machine-learning-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 64 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0433\u043e\u0020\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f" +--- + # Функции машинного обучения {#funktsii-mashinnogo-obucheniia} ## evalMLMethod (prediction) {#machine_learning_methods-evalmlmethod} diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index e52b14d26c6..6df366d129f 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: "\u041c\u0430\u0442\u0435\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Математические функции {#matematicheskie-funktsii} Все функции возвращают число типа Float64. Точность результата близка к максимально возможной, но результат может не совпадать с наиболее близким к соответствующему вещественному числу машинно представимым числом. diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index bb5371ff449..9367f3be00c 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 66 +toc_title: "\u041f\u0440\u043e\u0447\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Прочие функции {#other-functions} ## hostName() {#hostname} @@ -1468,7 +1473,7 @@ SELECT getSetting('custom_a'); ## isDecimalOverflow {#is-decimal-overflow} -Проверяет, находится ли число [Decimal](../../sql-reference/data-types/decimal.md#decimalp-s-decimal32s-decimal64s-decimal128s) вне собственной (или заданной) области значений. +Проверяет, находится ли число [Decimal](../../sql-reference/data-types/decimal.md) вне собственной (или заданной) области значений. **Синтаксис** @@ -1478,7 +1483,7 @@ isDecimalOverflow(d, [p]) **Параметры** -- `d` — число. [Decimal](../../sql-reference/data-types/decimal.md#decimalp-s-decimal32s-decimal64s-decimal128s). +- `d` — число. [Decimal](../../sql-reference/data-types/decimal.md). - `p` — точность. Необязательный параметр. Если опущен, используется исходная точность первого аргумента. Использование этого параметра может быть полезно для извлечения данных в другую СУБД или файл. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). **Возвращаемое значение** @@ -1515,7 +1520,7 @@ countDigits(x) **Параметры** -- `x` — [целое](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64) или [дробное](../../sql-reference/data-types/decimal.md#decimalp-s-decimal32s-decimal64s-decimal128s) число. +- `x` — [целое](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64) или [дробное](../../sql-reference/data-types/decimal.md) число. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index 21dcfeeb3c0..f3889504fa6 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 51 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0433\u0435\u043d\u0435\u0440\u0430\u0446\u0438\u0438\u0020\u043f\u0441\u0435\u0432\u0434\u043e\u0441\u043b\u0443\u0447\u0430\u0439\u043d\u044b\u0445\u0020\u0447\u0438\u0441\u0435\u043b" +--- + # Функции генерации псевдослучайных чисел {#functions-for-generating-pseudo-random-numbers} Используются не криптографические генераторы псевдослучайных чисел. diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md index c643bccf3e2..78033160396 100644 --- a/docs/ru/sql-reference/functions/rounding-functions.md +++ b/docs/ru/sql-reference/functions/rounding-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043e\u043a\u0440\u0443\u0433\u043b\u0435\u043d\u0438\u044f" +--- + # Функции округления {#funktsii-okrugleniia} ## floor(x\[, N\]) {#floorx-n} diff --git a/docs/ru/sql-reference/functions/splitting-merging-functions.md b/docs/ru/sql-reference/functions/splitting-merging-functions.md index bf4e76c3bb1..d451eabc407 100644 --- a/docs/ru/sql-reference/functions/splitting-merging-functions.md +++ b/docs/ru/sql-reference/functions/splitting-merging-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0440\u0430\u0437\u0431\u0438\u0435\u043d\u0438\u044f\u0020\u0438\u0020\u0441\u043b\u0438\u044f\u043d\u0438\u044f\u0020\u0441\u0442\u0440\u043e\u043a\u0020\u0438\u0020\u043c\u0430\u0441\u0441\u0438\u0432\u043e\u0432" +--- + # Функции разбиения и слияния строк и массивов {#funktsii-razbieniia-i-sliianiia-strok-i-massivov} ## splitByChar(separator, s) {#splitbycharseparator-s} diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 58ed582b399..cc488fb2d9c 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u043e\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u043c\u0438" +--- + # Функции для работы со строками {#funktsii-dlia-raboty-so-strokami} ## empty {#empty} diff --git a/docs/ru/sql-reference/functions/string-replace-functions.md b/docs/ru/sql-reference/functions/string-replace-functions.md index 8a2db9bf472..f334d6804f9 100644 --- a/docs/ru/sql-reference/functions/string-replace-functions.md +++ b/docs/ru/sql-reference/functions/string-replace-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u043e\u0438\u0441\u043a\u0430\u0020\u0438\u0020\u0437\u0430\u043c\u0435\u043d\u044b\u0020\u0432\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u0445" +--- + # Функции поиска и замены в строках {#funktsii-poiska-i-zameny-v-strokakh} ## replaceOne(haystack, pattern, replacement) {#replaceonehaystack-pattern-replacement} diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index 078a09a8aa4..d2f1119783b 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u043e\u0438\u0441\u043a\u0430\u0020\u0432\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u0445" +--- + # Функции поиска в строках {#funktsii-poiska-v-strokakh} Во всех функциях, поиск регистрозависимый по умолчанию. Существуют варианты функций для регистронезависимого поиска. diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 3bee1a3656f..773850b65ce 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u043e\u0432\u0430\u043d\u0438\u044f\u0020\u0442\u0438\u043f\u043e\u0432" +--- + # Функции преобразования типов {#funktsii-preobrazovaniia-tipov} ## Общие проблемы преобразования чисел {#numeric-conversion-issues} diff --git a/docs/ru/sql-reference/functions/url-functions.md b/docs/ru/sql-reference/functions/url-functions.md index 54b752d7c68..8f10a1ebd2b 100644 --- a/docs/ru/sql-reference/functions/url-functions.md +++ b/docs/ru/sql-reference/functions/url-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 54 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0055\u0052\u004c" +--- + # Функции для работы с URL {#funktsii-dlia-raboty-s-url} Все функции работают не по RFC - то есть, максимально упрощены ради производительности. diff --git a/docs/ru/sql-reference/functions/uuid-functions.md b/docs/ru/sql-reference/functions/uuid-functions.md index b2b567c6a06..389ce751ce0 100644 --- a/docs/ru/sql-reference/functions/uuid-functions.md +++ b/docs/ru/sql-reference/functions/uuid-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 53 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0055\u0055\u0049\u0044" +--- + # Функции для работы с UUID {#funktsii-dlia-raboty-s-uuid} ## generateUUIDv4 {#uuid-function-generate} diff --git a/docs/ru/sql-reference/functions/ym-dict-functions.md b/docs/ru/sql-reference/functions/ym-dict-functions.md index 7ac27c0d285..c3b04e4ab66 100644 --- a/docs/ru/sql-reference/functions/ym-dict-functions.md +++ b/docs/ru/sql-reference/functions/ym-dict-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u043e\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f\u043c\u0438\u0020\u042f\u043d\u0434\u0435\u043a\u0441\u002e\u041c\u0435\u0442\u0440\u0438\u043a\u0438" +--- + # Функции для работы со словарями Яндекс.Метрики {#ym-dict-functions} Чтобы указанные ниже функции работали, в конфиге сервера должны быть указаны пути и адреса для получения всех словарей Яндекс.Метрики. Словари загружаются при первом вызове любой из этих функций. Если справочники не удаётся загрузить - будет выкинуто исключение. diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 08594193d4c..3befb18687d 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: "\u041e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u044b" +--- + # Операторы {#operatory} Все операторы преобразуются в соответствующие функции на этапе парсинга запроса, с учётом их приоритетов и ассоциативности. diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 811539d60d3..a8fe0ccf642 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: COLUMN +toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u043e\u0020\u0441\u0442\u043e\u043b\u0431\u0446\u0430\u043c\u0438" --- # Манипуляции со столбцами {#manipuliatsii-so-stolbtsami} diff --git a/docs/ru/sql-reference/statements/alter/constraint.md b/docs/ru/sql-reference/statements/alter/constraint.md index bacdff9ff57..e26db208493 100644 --- a/docs/ru/sql-reference/statements/alter/constraint.md +++ b/docs/ru/sql-reference/statements/alter/constraint.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: CONSTRAINT +toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u0020\u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u043c\u0438" --- # Манипуляции с ограничениями (constraints) {#manipuliatsii-s-ogranicheniiami-constraints} diff --git a/docs/ru/sql-reference/statements/alter/index.md b/docs/ru/sql-reference/statements/alter/index.md index 035be934eb4..830c4a5745b 100644 --- a/docs/ru/sql-reference/statements/alter/index.md +++ b/docs/ru/sql-reference/statements/alter/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 36 +toc_priority: 35 toc_title: ALTER --- diff --git a/docs/ru/sql-reference/statements/alter/index/index.md b/docs/ru/sql-reference/statements/alter/index/index.md index 863f8b875dd..2cadbbe065e 100644 --- a/docs/ru/sql-reference/statements/alter/index/index.md +++ b/docs/ru/sql-reference/statements/alter/index/index.md @@ -1,7 +1,7 @@ --- toc_hidden_folder: true toc_priority: 42 -toc_title: INDEX +toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u0020\u0438\u043d\u0434\u0435\u043a\u0441\u0430\u043c\u0438" --- # Манипуляции с индексами {#manipuliatsii-s-indeksami} diff --git a/docs/ru/sql-reference/statements/alter/sample-by.md b/docs/ru/sql-reference/statements/alter/sample-by.md new file mode 100644 index 00000000000..2235e345a45 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/sample-by.md @@ -0,0 +1,20 @@ +--- +toc_priority: 41 +toc_title: SAMPLE BY +--- + +# Manipulating Sampling-Key Expressions {#manipulations-with-sampling-key-expressions} + +Синтаксис: + +``` sql +ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY SAMPLE BY new_expression +``` + +Команда меняет [ключ сэмплирования](../../../engines/table-engines/mergetree-family/mergetree.md) таблицы на `new_expression` (выражение или ряд выражений). + +Эта команда является упрощенной в том смысле, что она изменяет только метаданные. Первичный ключ должен содержать новый ключ сэмплирования. + +!!! note "Note" + Это работает только для таблиц в семействе [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) (включая +[реплицируемые](../../../engines/table-engines/mergetree-family/replication.md) таблицы). \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index 2aaf133130b..259ab893e63 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -1,5 +1,5 @@ --- -toc_priority: 42 +toc_priority: 40 toc_title: ATTACH --- diff --git a/docs/ru/sql-reference/statements/check-table.md b/docs/ru/sql-reference/statements/check-table.md index fa37b24851f..3dc135d87c6 100644 --- a/docs/ru/sql-reference/statements/check-table.md +++ b/docs/ru/sql-reference/statements/check-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 43 +toc_priority: 41 toc_title: CHECK --- diff --git a/docs/ru/sql-reference/statements/create/database.md b/docs/ru/sql-reference/statements/create/database.md index a6c8b81c8b8..e6c561f8e0b 100644 --- a/docs/ru/sql-reference/statements/create/database.md +++ b/docs/ru/sql-reference/statements/create/database.md @@ -1,6 +1,6 @@ --- -toc_priority: 1 -toc_title: База данных +toc_priority: 35 +toc_title: "\u0411\u0430\u0437\u0430\u0020\u0434\u0430\u043d\u043d\u044b\u0445" --- # CREATE DATABASE {#query-language-create-database} diff --git a/docs/ru/sql-reference/statements/create/dictionary.md b/docs/ru/sql-reference/statements/create/dictionary.md index da9443e1b3a..a20dc812e02 100644 --- a/docs/ru/sql-reference/statements/create/dictionary.md +++ b/docs/ru/sql-reference/statements/create/dictionary.md @@ -1,6 +1,6 @@ --- -toc_priority: 4 -toc_title: Словарь +toc_priority: 38 +toc_title: "\u0421\u043b\u043e\u0432\u0430\u0440\u044c" --- # CREATE DICTIONARY {#create-dictionary-query} diff --git a/docs/ru/sql-reference/statements/create/index.md b/docs/ru/sql-reference/statements/create/index.md index f6399cfba11..28ddce2afe3 100644 --- a/docs/ru/sql-reference/statements/create/index.md +++ b/docs/ru/sql-reference/statements/create/index.md @@ -1,7 +1,7 @@ --- toc_folder_title: CREATE -toc_priority: 35 -toc_title: Обзор +toc_priority: 34 +toc_title: "\u041e\u0431\u0437\u043e\u0440" --- # Запросы CREATE {#create-queries} diff --git a/docs/ru/sql-reference/statements/create/quota.md b/docs/ru/sql-reference/statements/create/quota.md index 6351de2d38a..fe18869bf2e 100644 --- a/docs/ru/sql-reference/statements/create/quota.md +++ b/docs/ru/sql-reference/statements/create/quota.md @@ -1,6 +1,6 @@ --- -toc_priority: 8 -toc_title: Квота +toc_priority: 42 +toc_title: "\u041a\u0432\u043e\u0442\u0430" --- # CREATE QUOTA {#create-quota-statement} diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index b8c0fc2b453..b9e529fb213 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -1,6 +1,6 @@ --- -toc_priority: 6 -toc_title: Роль +toc_priority: 40 +toc_title: "\u0420\u043e\u043b\u044c" --- # CREATE ROLE {#create-role-statement} diff --git a/docs/ru/sql-reference/statements/create/row-policy.md b/docs/ru/sql-reference/statements/create/row-policy.md index 7ec28761452..a62e275a046 100644 --- a/docs/ru/sql-reference/statements/create/row-policy.md +++ b/docs/ru/sql-reference/statements/create/row-policy.md @@ -1,6 +1,6 @@ --- -toc_priority: 7 -toc_title: Политика доступа +toc_priority: 41 +toc_title: "\u041f\u043e\u043b\u0438\u0442\u0438\u043a\u0430\u0020\u0434\u043e\u0441\u0442\u0443\u043f\u0430" --- # CREATE ROW POLICY {#create-row-policy-statement} diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index 70c90b53565..9d525023af2 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -1,6 +1,6 @@ --- -toc_priority: 9 -toc_title: Профиль настроек +toc_priority: 43 +toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u044c\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" --- # CREATE SETTINGS PROFILE {#create-settings-profile-statement} diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index c6093bbc6de..d54ec189a1a 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -1,6 +1,6 @@ --- -toc_priority: 2 -toc_title: Таблица +toc_priority: 36 +toc_title: "\u0422\u0430\u0431\u043b\u0438\u0446\u0430" --- # CREATE TABLE {#create-table-query} diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md index 3c04b4df86c..e7af1659a1b 100644 --- a/docs/ru/sql-reference/statements/create/user.md +++ b/docs/ru/sql-reference/statements/create/user.md @@ -1,6 +1,6 @@ --- -toc_priority: 5 -toc_title: Пользователь +toc_priority: 39 +toc_title: "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c" --- # CREATE USER {#create-user-statement} diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index caa3d04659e..891e33bc9b3 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -1,6 +1,6 @@ --- -toc_priority: 3 -toc_title: Представление +toc_priority: 37 +toc_title: "\u041f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435" --- # CREATE VIEW {#create-view} diff --git a/docs/ru/sql-reference/statements/describe-table.md b/docs/ru/sql-reference/statements/describe-table.md index 5b38dca5833..64ed61de232 100644 --- a/docs/ru/sql-reference/statements/describe-table.md +++ b/docs/ru/sql-reference/statements/describe-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 44 +toc_priority: 42 toc_title: DESCRIBE --- diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 172e86179c8..00d0a4b20c6 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -1,5 +1,5 @@ --- -toc_priority: 45 +toc_priority: 43 toc_title: DETACH --- diff --git a/docs/ru/sql-reference/statements/drop.md b/docs/ru/sql-reference/statements/drop.md index 22e553cfdac..514a92db91f 100644 --- a/docs/ru/sql-reference/statements/drop.md +++ b/docs/ru/sql-reference/statements/drop.md @@ -1,110 +1,100 @@ --- -toc_priority: 46 +toc_priority: 44 toc_title: DROP --- # DROP {#drop} -Удаляет существующий объект. -Если указано `IF EXISTS` - не выдавать ошибку, если объекта не существует. +Удаляет существующий объект. Если указано `IF EXISTS` - не выдавать ошибку, если объекта не существует. ## DROP DATABASE {#drop-database} +Удаляет все таблицы в базе данных `db`, затем удаляет саму базу данных `db`. + +Синтаксис: + ``` sql DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Удаляет все таблицы в базе данных db, затем удаляет саму базу данных db. - - ## DROP TABLE {#drop-table} +Удаляет таблицу. + +Синтаксис: + ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Удаляет таблицу. - - ## DROP DICTIONARY {#drop-dictionary} +Удаляет словарь. + +Синтаксис: + ``` sql DROP DICTIONARY [IF EXISTS] [db.]name ``` -Удаляет словарь. - - ## DROP USER {#drop-user-statement} Удаляет пользователя. -### Синтаксис {#drop-user-syntax} +Синтаксис: ```sql DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP ROLE {#drop-role-statement} -Удаляет роль. +Удаляет роль. При удалении роль отзывается у всех объектов системы доступа, которым она присвоена. -При удалении роль отзывается у всех объектов системы доступа, которым она присвоена. - -### Синтаксис {#drop-role-syntax} +Синтаксис: ```sql DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP ROW POLICY {#drop-row-policy-statement} -Удаляет политику доступа к строкам. +Удаляет политику доступа к строкам. При удалении политика отзывается у всех объектов системы доступа, которым она присвоена. -При удалении политика отзывается у всех объектов системы доступа, которым она присвоена. - -### Синтаксис {#drop-row-policy-syntax} +Синтаксис: ``` sql DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] ``` - ## DROP QUOTA {#drop-quota-statement} -Удаляет квоту. +Удаляет квоту. При удалении квота отзывается у всех объектов системы доступа, которым она присвоена. -При удалении квота отзывается у всех объектов системы доступа, которым она присвоена. - -### Синтаксис {#drop-quota-syntax} +Синтаксис: ``` sql DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP SETTINGS PROFILE {#drop-settings-profile-statement} -Удаляет профиль настроек. +Удаляет профиль настроек. При удалении профиль отзывается у всех объектов системы доступа, которым он присвоен. -При удалении профиль отзывается у всех объектов системы доступа, которым он присвоен. - -### Синтаксис {#drop-settings-profile-syntax} +Синтаксис: ``` sql DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP VIEW {#drop-view} +Удаляет представление. Представления могут быть удалены и командой `DROP TABLE`, но команда `DROP VIEW` проверяет, что `[db.]name` является представлением. + +Синтаксис: + ``` sql DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Удаляет представление. Представления могут быть удалены и командой `DROP TABLE`, но команда `DROP VIEW` проверяет, что `[db.]name` является представлением. - - -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/exists.md b/docs/ru/sql-reference/statements/exists.md index 865d23ad622..0b2fd69273c 100644 --- a/docs/ru/sql-reference/statements/exists.md +++ b/docs/ru/sql-reference/statements/exists.md @@ -1,5 +1,5 @@ --- -toc_priority: 47 +toc_priority: 45 toc_title: EXISTS --- diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 8eea84ac594..d38e2ea38a0 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: GRANT +--- + # GRANT - Присваивает [привилегии](#grant-privileges) пользователям или ролям ClickHouse. @@ -79,6 +84,7 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION - `ALTER RENAME COLUMN` - `ALTER INDEX` - `ALTER ORDER BY` + - `ALTER SAMPLE BY` - `ALTER ADD INDEX` - `ALTER DROP INDEX` - `ALTER MATERIALIZE INDEX` @@ -264,6 +270,7 @@ GRANT INSERT(x,y) ON db.table TO john - `ALTER RENAME COLUMN`. Уровень: `COLUMN`. Алиасы: `RENAME COLUMN` - `ALTER INDEX`. Уровень: `GROUP`. Алиасы: `INDEX` - `ALTER ORDER BY`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY ORDER BY`, `MODIFY ORDER BY` + - `ALTER SAMPLE BY`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY SAMPLE BY`, `MODIFY SAMPLE BY` - `ALTER ADD INDEX`. Уровень: `TABLE`. Алиасы: `ADD INDEX` - `ALTER DROP INDEX`. Уровень: `TABLE`. Алиасы: `DROP INDEX` - `ALTER MATERIALIZE INDEX`. Уровень: `TABLE`. Алиасы: `MATERIALIZE INDEX` diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index 281b708f3f9..8ea7c83bec8 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -1,5 +1,5 @@ --- -toc_priority: 34 +toc_priority: 33 toc_title: INSERT INTO --- diff --git a/docs/ru/sql-reference/statements/kill.md b/docs/ru/sql-reference/statements/kill.md index dd2f24ffa27..e2556a7f782 100644 --- a/docs/ru/sql-reference/statements/kill.md +++ b/docs/ru/sql-reference/statements/kill.md @@ -1,5 +1,5 @@ --- -toc_priority: 48 +toc_priority: 46 toc_title: KILL --- diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index dc70d86a1a0..9b94c31a8f7 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -1,5 +1,5 @@ --- -toc_priority: 49 +toc_priority: 47 toc_title: OPTIMIZE --- diff --git a/docs/ru/sql-reference/statements/rename.md b/docs/ru/sql-reference/statements/rename.md index 9f6069d8a94..94bf3c682a1 100644 --- a/docs/ru/sql-reference/statements/rename.md +++ b/docs/ru/sql-reference/statements/rename.md @@ -1,5 +1,5 @@ --- -toc_priority: 50 +toc_priority: 48 toc_title: RENAME --- diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md index 1d2928bb76e..339746b8591 100644 --- a/docs/ru/sql-reference/statements/revoke.md +++ b/docs/ru/sql-reference/statements/revoke.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: REVOKE +--- + # REVOKE Отзывает привилегии у пользователей или ролей. diff --git a/docs/ru/sql-reference/statements/select/array-join.md b/docs/ru/sql-reference/statements/select/array-join.md index f8f11ba1b17..a3abf9e5e2e 100644 --- a/docs/ru/sql-reference/statements/select/array-join.md +++ b/docs/ru/sql-reference/statements/select/array-join.md @@ -1,3 +1,7 @@ +--- +toc_title: ARRAY JOIN +--- + # Секция ARRAY JOIN {#select-array-join-clause} Типовая операция для таблиц, содержащих столбец-массив — произвести новую таблицу, которая будет иметь столбец с каждым отдельным элементом массивов из изначального столбца, в то время как значения других столбцов дублируются. Это основной сценарий использования секции `ARRAY JOIN`. diff --git a/docs/ru/sql-reference/statements/select/distinct.md b/docs/ru/sql-reference/statements/select/distinct.md index 62e2e25b7e5..9d620079f6b 100644 --- a/docs/ru/sql-reference/statements/select/distinct.md +++ b/docs/ru/sql-reference/statements/select/distinct.md @@ -1,3 +1,7 @@ +--- +toc_title: DISTINCT +--- + # Секция DISTINCT {#select-distinct} Если указан `SELECT DISTINCT`, то в результате запроса останутся только уникальные строки. Таким образом, из всех наборов полностью совпадающих строк в результате останется только одна строка. diff --git a/docs/ru/sql-reference/statements/select/format.md b/docs/ru/sql-reference/statements/select/format.md index dad0ef0d62c..18972ddd82b 100644 --- a/docs/ru/sql-reference/statements/select/format.md +++ b/docs/ru/sql-reference/statements/select/format.md @@ -1,3 +1,7 @@ +--- +toc_title: FORMAT +--- + # Секция FORMAT {#format-clause} ClickHouse поддерживает широкий спектр [форматов сериализации](../../../interfaces/formats.md) это может быть использовано, в частности, для результатов запросов. Существует несколько способов выбора формата для `SELECT`, один из них заключается в том, чтобы указать `FORMAT format` в конце запроса, чтобы получить результирующие данные в любом конкретном формате. diff --git a/docs/ru/sql-reference/statements/select/from.md b/docs/ru/sql-reference/statements/select/from.md index ac0ab1dcd3f..491bbfe892b 100644 --- a/docs/ru/sql-reference/statements/select/from.md +++ b/docs/ru/sql-reference/statements/select/from.md @@ -1,3 +1,7 @@ +--- +toc_title: FROM +--- + # Секция FROM {#select-from} В секции `FROM` указывается источник, из которого будут читаться данные: diff --git a/docs/ru/sql-reference/statements/select/group-by.md b/docs/ru/sql-reference/statements/select/group-by.md index 9581f477af5..a0454ef1d91 100644 --- a/docs/ru/sql-reference/statements/select/group-by.md +++ b/docs/ru/sql-reference/statements/select/group-by.md @@ -1,3 +1,7 @@ +--- +toc_title: GROUP BY +--- + # Секция GROUP BY {#select-group-by-clause} Секция `GROUP BY` переключает `SELECT` запрос в режим агрегации, который работает следующим образом: diff --git a/docs/ru/sql-reference/statements/select/having.md b/docs/ru/sql-reference/statements/select/having.md index 83f58c5566f..dc701df906f 100644 --- a/docs/ru/sql-reference/statements/select/having.md +++ b/docs/ru/sql-reference/statements/select/having.md @@ -1,3 +1,7 @@ +--- +toc_title: HAVING +--- + # Секция HAVING {#having-clause} Позволяет фильтровать результаты агрегации, полученные с помощью [GROUP BY](group-by.md). Разница с [WHERE](where.md) в том, что `WHERE` выполняется перед агрегацией, в то время как `HAVING` выполняется после него. diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md index b735d37118c..f5fe2788370 100644 --- a/docs/ru/sql-reference/statements/select/index.md +++ b/docs/ru/sql-reference/statements/select/index.md @@ -1,6 +1,8 @@ --- -toc_priority: 33 -toc_title: SELECT +title: "\u0421\u0438\u043d\u0442\u0430\u043a\u0441\u0438\u0441\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432\u0020\u0053\u0045\u004c\u0045\u0043\u0054" +toc_folder_title: SELECT +toc_priority: 32 +toc_title: "\u041e\u0431\u0437\u043e\u0440" --- # Синтаксис запросов SELECT {#select-queries-syntax} diff --git a/docs/ru/sql-reference/statements/select/into-outfile.md b/docs/ru/sql-reference/statements/select/into-outfile.md index 0f5cf01e9d1..f956903f8b4 100644 --- a/docs/ru/sql-reference/statements/select/into-outfile.md +++ b/docs/ru/sql-reference/statements/select/into-outfile.md @@ -1,3 +1,7 @@ +--- +toc_title: INTO OUTFILE +--- + # Секция INTO OUTFILE {#into-outfile-clause} Чтобы перенаправить вывод `SELECT` запроса в указанный файл на стороне клиента, добавьте к нему секцию `INTO OUTFILE filename` (где filename — строковый литерал). diff --git a/docs/ru/sql-reference/statements/select/join.md b/docs/ru/sql-reference/statements/select/join.md index 6f1b5e2cde6..c5548d74156 100644 --- a/docs/ru/sql-reference/statements/select/join.md +++ b/docs/ru/sql-reference/statements/select/join.md @@ -1,3 +1,7 @@ +--- +toc_title: JOIN +--- + # Секция JOIN {#select-join} Join создаёт новую таблицу путем объединения столбцов из одной или нескольких таблиц с использованием общих для каждой из них значений. Это обычная операция в базах данных с поддержкой SQL, которая соответствует join из [реляционной алгебры](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators). Частный случай соединения одной таблицы часто называют «self-join». diff --git a/docs/ru/sql-reference/statements/select/limit-by.md b/docs/ru/sql-reference/statements/select/limit-by.md index ea5d467ae4f..fba81c023b5 100644 --- a/docs/ru/sql-reference/statements/select/limit-by.md +++ b/docs/ru/sql-reference/statements/select/limit-by.md @@ -1,3 +1,7 @@ +--- +toc_title: LIMIT BY +--- + # Секция LIMIT BY {#limit-by-clause} Запрос с секцией `LIMIT n BY expressions` выбирает первые `n` строк для каждого отличного значения `expressions`. Ключ `LIMIT BY` может содержать любое количество [выражений](../../syntax.md#syntax-expressions). diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md index 1c8595d8e0c..c2a02b1a436 100644 --- a/docs/ru/sql-reference/statements/select/prewhere.md +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -1,3 +1,7 @@ +--- +toc_title: PREWHERE +--- + # Секция PREWHERE {#prewhere-clause} Prewhere — это оптимизация для более эффективного применения фильтрации. Она включена по умолчанию, даже если секция `PREWHERE` явно не указана. В этом случае работает автоматическое перемещение части выражения из [WHERE](where.md) до стадии prewhere. Роль секции `PREWHERE` только для управления этой оптимизацией, если вы думаете, что знаете, как сделать перемещение условия лучше, чем это происходит по умолчанию. diff --git a/docs/ru/sql-reference/statements/select/sample.md b/docs/ru/sql-reference/statements/select/sample.md index ca6b49c9ad6..a71e8256f0f 100644 --- a/docs/ru/sql-reference/statements/select/sample.md +++ b/docs/ru/sql-reference/statements/select/sample.md @@ -1,3 +1,7 @@ +--- +toc_title: SAMPLE +--- + # Секция SAMPLE {#select-sample-clause} Секция `SAMPLE` позволяет выполнять запросы приближённо. Например, чтобы посчитать статистику по всем визитам, можно обработать 1/10 всех визитов и результат домножить на 10. diff --git a/docs/ru/sql-reference/statements/select/union-all.md b/docs/ru/sql-reference/statements/select/union-all.md index bffd667fa1d..b9d1f485a7b 100644 --- a/docs/ru/sql-reference/statements/select/union-all.md +++ b/docs/ru/sql-reference/statements/select/union-all.md @@ -1,3 +1,7 @@ +--- +toc_title: UNION ALL +--- + # Секция UNION ALL {#union-all-clause} Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример: diff --git a/docs/ru/sql-reference/statements/select/where.md b/docs/ru/sql-reference/statements/select/where.md index 63d081db43d..8cb8e634303 100644 --- a/docs/ru/sql-reference/statements/select/where.md +++ b/docs/ru/sql-reference/statements/select/where.md @@ -1,3 +1,7 @@ +--- +toc_title: WHERE +--- + # Секция WHERE {#select-where} Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#prewhere-clause). Обычно, это выражение с логическими операторами. diff --git a/docs/ru/sql-reference/statements/select/with.md b/docs/ru/sql-reference/statements/select/with.md index a5be733866f..4feae232bd7 100644 --- a/docs/ru/sql-reference/statements/select/with.md +++ b/docs/ru/sql-reference/statements/select/with.md @@ -1,3 +1,7 @@ +--- +toc_title: WITH +--- + # Секция WITH {#sektsiia-with} Данная секция представляет собой [Common Table Expressions](https://ru.wikipedia.org/wiki/Иерархические_и_рекурсивные_запросы_в_SQL), то есть позволяет использовать результаты выражений из секции `WITH` в остальной части `SELECT` запроса. diff --git a/docs/ru/sql-reference/statements/set-role.md b/docs/ru/sql-reference/statements/set-role.md index 5d840fcddb7..ccbef41aa9b 100644 --- a/docs/ru/sql-reference/statements/set-role.md +++ b/docs/ru/sql-reference/statements/set-role.md @@ -1,5 +1,5 @@ --- -toc_priority: 52 +toc_priority: 50 toc_title: SET ROLE --- diff --git a/docs/ru/sql-reference/statements/set.md b/docs/ru/sql-reference/statements/set.md index 0e12e2ee7bc..b60dfcf8324 100644 --- a/docs/ru/sql-reference/statements/set.md +++ b/docs/ru/sql-reference/statements/set.md @@ -1,5 +1,5 @@ --- -toc_priority: 51 +toc_priority: 49 toc_title: SET --- diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 4b226765632..4516a401de9 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: SHOW +--- + # SHOW Queries {#show-queries} ## SHOW CREATE TABLE {#show-create-table} diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 1ba3a7555b9..4780e9b613f 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: SYSTEM +--- + # Запросы SYSTEM {#query-language-system} - [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) diff --git a/docs/ru/sql-reference/statements/truncate.md b/docs/ru/sql-reference/statements/truncate.md index f8806496e48..4909d349658 100644 --- a/docs/ru/sql-reference/statements/truncate.md +++ b/docs/ru/sql-reference/statements/truncate.md @@ -1,5 +1,5 @@ --- -toc_priority: 53 +toc_priority: 51 toc_title: TRUNCATE --- diff --git a/docs/ru/sql-reference/statements/use.md b/docs/ru/sql-reference/statements/use.md index 84b0f5ed13f..c84329ea5ff 100644 --- a/docs/ru/sql-reference/statements/use.md +++ b/docs/ru/sql-reference/statements/use.md @@ -1,5 +1,5 @@ --- -toc_priority: 54 +toc_priority: 52 toc_title: USE --- diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index 5c819002cbe..ca73d3a137e 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -1,3 +1,8 @@ +--- +toc_priority: 31 +toc_title: "\u0421\u0438\u043d\u0442\u0430\u043a\u0441\u0438\u0441" +--- + # Синтаксис {#sintaksis} В системе есть два вида парсеров: полноценный парсер SQL (recursive descent parser) и парсер форматов данных (быстрый потоковый парсер). diff --git a/docs/ru/sql-reference/table-functions/file.md b/docs/ru/sql-reference/table-functions/file.md index 4581fa081d7..d3e6e106125 100644 --- a/docs/ru/sql-reference/table-functions/file.md +++ b/docs/ru/sql-reference/table-functions/file.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: file +--- + # file {#file} Создаёт таблицу из файла. Данная табличная функция похожа на табличные функции [file](file.md) и [hdfs](hdfs.md). diff --git a/docs/ru/sql-reference/table-functions/generate.md b/docs/ru/sql-reference/table-functions/generate.md index 9e6d36b2a4b..47b7e43bc86 100644 --- a/docs/ru/sql-reference/table-functions/generate.md +++ b/docs/ru/sql-reference/table-functions/generate.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: generateRandom +--- + # generateRandom {#generaterandom} Генерирует случайные данные с заданной схемой. diff --git a/docs/ru/sql-reference/table-functions/hdfs.md b/docs/ru/sql-reference/table-functions/hdfs.md index e90f27a2eca..6edd70b7b1b 100644 --- a/docs/ru/sql-reference/table-functions/hdfs.md +++ b/docs/ru/sql-reference/table-functions/hdfs.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: hdfs +--- + # hdfs {#hdfs} Создаёт таблицу из файла в HDFS. Данная табличная функция похожа на табличные функции [url](url.md) и [file](file.md). diff --git a/docs/ru/sql-reference/table-functions/input.md b/docs/ru/sql-reference/table-functions/input.md index 399268f9af6..96cf7515d52 100644 --- a/docs/ru/sql-reference/table-functions/input.md +++ b/docs/ru/sql-reference/table-functions/input.md @@ -1,3 +1,8 @@ +--- +toc_priority: 46 +toc_title: input +--- + # input {#input} `input(structure)` - табличная функция, позволяющая эффективно преобразовывать и вставлять отправленные на сервер данные, diff --git a/docs/ru/sql-reference/table-functions/jdbc.md b/docs/ru/sql-reference/table-functions/jdbc.md index 20ef0d1f107..d388262606f 100644 --- a/docs/ru/sql-reference/table-functions/jdbc.md +++ b/docs/ru/sql-reference/table-functions/jdbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: jdbc +--- + # jdbc {#jdbc} `jdbc(jdbc_connection_uri, schema, table)` - возвращает таблицу, соединение с которой происходит через JDBC-драйвер. diff --git a/docs/ru/sql-reference/table-functions/merge.md b/docs/ru/sql-reference/table-functions/merge.md index d4e66391382..0822fdfe535 100644 --- a/docs/ru/sql-reference/table-functions/merge.md +++ b/docs/ru/sql-reference/table-functions/merge.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: merge +--- + # merge {#merge} `merge(db_name, 'tables_regexp')` - создаёт временную таблицу типа Merge. Подробнее смотрите раздел «Движки таблиц, Merge». diff --git a/docs/ru/sql-reference/table-functions/mysql.md b/docs/ru/sql-reference/table-functions/mysql.md index 99d82022df4..21841eee67a 100644 --- a/docs/ru/sql-reference/table-functions/mysql.md +++ b/docs/ru/sql-reference/table-functions/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: mysql +--- + # mysql {#mysql} Позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом MySQL сервере. diff --git a/docs/ru/sql-reference/table-functions/numbers.md b/docs/ru/sql-reference/table-functions/numbers.md index 79d01dd0b92..005f400e082 100644 --- a/docs/ru/sql-reference/table-functions/numbers.md +++ b/docs/ru/sql-reference/table-functions/numbers.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: numbers +--- + # numbers {#numbers} `numbers(N)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `0` до `N-1`. diff --git a/docs/ru/sql-reference/table-functions/odbc.md b/docs/ru/sql-reference/table-functions/odbc.md index 38da5066cbd..19203123840 100644 --- a/docs/ru/sql-reference/table-functions/odbc.md +++ b/docs/ru/sql-reference/table-functions/odbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: odbc +--- + # odbc {#table-functions-odbc} Возвращает таблицу, подключенную через [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). diff --git a/docs/ru/sql-reference/table-functions/remote.md b/docs/ru/sql-reference/table-functions/remote.md index 944500b57a0..901317a805d 100644 --- a/docs/ru/sql-reference/table-functions/remote.md +++ b/docs/ru/sql-reference/table-functions/remote.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: remote +--- + # remote, remoteSecure {#remote-remotesecure} Позволяет обратиться к удалённым серверам без создания таблицы типа `Distributed`. diff --git a/docs/ru/sql-reference/table-functions/url.md b/docs/ru/sql-reference/table-functions/url.md index 4c49bc76751..0cd7c24c663 100644 --- a/docs/ru/sql-reference/table-functions/url.md +++ b/docs/ru/sql-reference/table-functions/url.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: url +--- + # url {#url} `url(URL, format, structure)` - возвращает таблицу со столбцами, указанными в diff --git a/docs/ru/whats-new/index.md b/docs/ru/whats-new/index.md index a8c908088d0..b8ba6133454 100644 --- a/docs/ru/whats-new/index.md +++ b/docs/ru/whats-new/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: "\u0427\u0442\u043E \u043D\u043E\u0432\u043E\u0433\u043E?" -toc_priority: 72 +toc_priority: 82 --- diff --git a/docs/ru/whats-new/security-changelog.md b/docs/ru/whats-new/security-changelog.md index ad55e1eeb51..1f46535833d 100644 --- a/docs/ru/whats-new/security-changelog.md +++ b/docs/ru/whats-new/security-changelog.md @@ -1,3 +1,8 @@ +--- +toc_priority: 76 +toc_title: Security Changelog +--- + # Security Changelog {#security-changelog} ## Исправлено в релизе 19.14.3.3, 2019-09-10 {#ispravleno-v-relize-19-14-3-3-2019-09-10} diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3cd584c0e55..ace509d6691 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -218,6 +218,8 @@ private: QueryFuzzer fuzzer; int query_fuzzer_runs = 0; + std::optional suggest; + /// We will format query_id in interactive mode in various ways, the default is just to print Query id: ... std::vector> query_id_formats; @@ -577,10 +579,11 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); + suggest.emplace(); if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false)) { /// Load suggestion data from the server. - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); + suggest->load(connection_parameters, config().getInt("suggestion_limit")); } /// Load command history if present. @@ -607,7 +610,7 @@ private: highlight_callback = highlight; ReplxxLineReader lr( - Suggest::instance(), + *suggest, history_file, config().has("multiline"), query_extenders, @@ -615,7 +618,7 @@ private: highlight_callback); #elif defined(USE_READLINE) && USE_READLINE - ReadlineLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters); + ReadlineLineReader lr(*suggest, history_file, config().has("multiline"), query_extenders, query_delimiters); #else LineReader lr(history_file, config().has("multiline"), query_extenders, query_delimiters); #endif @@ -2324,6 +2327,8 @@ public: ("log-level", po::value(), "client log level") ("server_logs_file", po::value(), "put server logs into specified file") ("query-fuzzer-runs", po::value()->default_value(0), "query fuzzer runs") + ("opentelemetry-traceparent", po::value(), "OpenTelemetry traceparent header as described by W3C Trace Context recommendation") + ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ; Settings cmd_settings; @@ -2492,6 +2497,25 @@ public: ignore_error = true; } + if (options.count("opentelemetry-traceparent")) + { + std::string traceparent = options["opentelemetry-traceparent"].as(); + std::string error; + if (!context.getClientInfo().parseTraceparentHeader( + traceparent, error)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot parse OpenTelemetry traceparent '{}': {}", + traceparent, error); + } + } + + if (options.count("opentelemetry-tracestate")) + { + context.getClientInfo().opentelemetry_tracestate = + options["opentelemetry-tracestate"].as(); + } + argsToConfig(common_arguments, config(), 100); clearPasswordFromCommandLine(argc, argv); diff --git a/programs/client/Suggest.h b/programs/client/Suggest.h index b13289ac322..03332088cbe 100644 --- a/programs/client/Suggest.h +++ b/programs/client/Suggest.h @@ -18,10 +18,11 @@ namespace ErrorCodes class Suggest : public LineReader::Suggest, boost::noncopyable { public: - static Suggest & instance() + Suggest(); + ~Suggest() { - static Suggest instance; - return instance; + if (loading_thread.joinable()) + loading_thread.join(); } void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit); @@ -30,12 +31,6 @@ public: static constexpr int MIN_SERVER_REVISION = 54406; private: - Suggest(); - ~Suggest() - { - if (loading_thread.joinable()) - loading_thread.join(); - } void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit); void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query); diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 2d019f81367..b9dde555788 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include @@ -47,9 +48,9 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; extern const int SYNTAX_ERROR; extern const int CANNOT_LOAD_CONFIG; + extern const int FILE_ALREADY_EXISTS; } @@ -121,31 +122,43 @@ void LocalServer::tryInitPath() } else { - // Default unique path in the system temporary directory. - const auto tmp = std::filesystem::temp_directory_path(); - const auto default_path = tmp - / fmt::format("clickhouse-local-{}", getpid()); + // The path is not provided explicitly - use a unique path in the system temporary directory + // (or in the current dir if temporary don't exist) + Poco::Logger * log = &logger(); + std::filesystem::path parent_folder; + std::filesystem::path default_path; + + try + { + // try to guess a tmp folder name, and check if it's a directory (throw exception otherwise) + parent_folder = std::filesystem::temp_directory_path(); + + } + catch (const std::filesystem::filesystem_error& e) + { + // tmp folder don't exists? misconfiguration? chroot? + LOG_DEBUG(log, "Can not get temporary folder: {}", e.what()); + parent_folder = std::filesystem::current_path(); + + std::filesystem::is_directory(parent_folder); // that will throw an exception if it's not a directory + LOG_DEBUG(log, "Will create working directory inside current directory: {}", parent_folder.string()); + } + + /// we can have another clickhouse-local running simultaneously, even with the same PID (for ex. - several dockers mounting the same folder) + /// or it can be some leftovers from other clickhouse-local runs + /// as we can't accurately distinguish those situations we don't touch any existent folders + /// we just try to pick some free name for our working folder + + default_path = parent_folder / fmt::format("clickhouse-local-{}-{}-{}", getpid(), time(nullptr), randomSeed()); if (exists(default_path)) - { - // This is a directory that is left by a previous run of - // clickhouse-local that had the same pid and did not complete - // correctly. Remove it, with an additional sanity check. - if (!std::filesystem::equivalent(default_path.parent_path(), tmp)) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "The temporary directory of clickhouse-local '{}' is not" - " inside the system temporary directory '{}'. Will not delete" - " it", default_path.string(), tmp.string()); - } - - remove_all(default_path); - } + throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Unsuccessful attempt to create working directory: {} exist!", default_path.string()); create_directory(default_path); temporary_directory_to_delete = default_path; path = default_path.string(); + LOG_DEBUG(log, "Working directory created: {}", path); } if (path.back() != '/') @@ -438,23 +451,12 @@ void LocalServer::setupUsers() void LocalServer::cleanup() { - // Delete the temporary directory if needed. Just in case, check that it is - // in the system temporary directory, not to delete user data if there is a - // bug. + // Delete the temporary directory if needed. if (temporary_directory_to_delete) { - const auto tmp = std::filesystem::temp_directory_path(); const auto dir = *temporary_directory_to_delete; temporary_directory_to_delete.reset(); - - if (!std::filesystem::equivalent(dir.parent_path(), tmp)) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "The temporary directory of clickhouse-local '{}' is not inside" - " the system temporary directory '{}'. Will not delete it", - dir.string(), tmp.string()); - } - + LOG_DEBUG(&logger(), "Removing temporary directory: {}", dir.string()); remove_all(dir); } } diff --git a/programs/server/config.xml b/programs/server/config.xml index 5bdec5377fd..e17b59671af 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -270,7 +270,7 @@ This parameter is mandatory and cannot be empty. roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. - If any of the listed roles is not defined locally at the time of authentication, the authenthication attept + If any of the listed roles is not defined locally at the time of authentication, the authenthication attempt will fail as if the provided password was incorrect. Example: @@ -628,6 +628,31 @@ 60000 + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + + + diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 61ab4c8002d..0459022cb1a 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -100,7 +100,7 @@ namespace if (res & alter_table) res |= alter_view; - /// CREATE TABLE (on any database/table) => CREATE_TEMPORARY_TABLE (global) + /// CREATE TABLE (on any database/table) => CREATE_TEMPORARY_TABLE (global) static const AccessFlags create_temporary_table = AccessType::CREATE_TEMPORARY_TABLE; if ((level == 0) && (max_flags_with_children & create_table)) res |= create_temporary_table; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8dd219e07d7..58821e7de4b 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -585,7 +585,7 @@ void IAccessStorage::throwInvalidPassword() void IAccessStorage::throwCannotAuthenticate(const String & user_name) { - /// We use the same message for all authentification failures because we don't want to give away any unnecessary information for security reasons, + /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons, /// only the log will show the exact reason. throw Exception(user_name + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED); } diff --git a/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp b/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp index e338b060b12..6f4f254ae8f 100644 --- a/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp +++ b/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp @@ -31,7 +31,7 @@ AggregateFunctionPtr createAggregateFunctionRate(const std::string & name, const void registerAggregateFunctionRate(AggregateFunctionFactory & factory) { - factory.registerFunction("boundingRatio", createAggregateFunctionRate, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("boundingRatio", createAggregateFunctionRate); } } diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.h b/src/AggregateFunctions/AggregateFunctionGroupArray.h index 83e096c797b..02b9003eb96 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h @@ -296,7 +296,7 @@ public: { typename ColumnVector::Container & data_to = assert_cast &>(arr_to.getData()).getData(); if constexpr (is_big_int_v) - // is data_to empty? we should probaly use std::vector::insert then + // is data_to empty? we should probably use std::vector::insert then for (auto it = this->data(place).value.begin(); it != this->data(place).value.end(); it++) data_to.push_back(*it); else diff --git a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp index 20472279dba..796ff028424 100644 --- a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp +++ b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp @@ -45,7 +45,7 @@ AggregateFunctionPtr createAggregateFunctionRankCorrelation(const std::string & void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory & factory) { - factory.registerFunction("rankCorr", createAggregateFunctionRankCorrelation, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("rankCorr", createAggregateFunctionRankCorrelation); } } diff --git a/src/AggregateFunctions/AggregateFunctionRetention.cpp b/src/AggregateFunctions/AggregateFunctionRetention.cpp index 4497703c550..c9d475c78f3 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.cpp +++ b/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -32,7 +32,7 @@ AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, void registerAggregateFunctionRetention(AggregateFunctionFactory & factory) { - factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("retention", createAggregateFunctionRetention); } } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index a2c36e43488..58fc9e5b5b9 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -47,6 +47,6 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & nam void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index d260a6be980..0aef8f3ee2a 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -20,9 +20,17 @@ namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } +#if defined(OS_DARWIN) +extern "C" +{ + double lgammal_r(double x, int * signgamp); +} +#endif + + namespace DB { @@ -98,7 +106,7 @@ struct AggregateFunctionStudentTTestData final Float64 getSSquared() const { - /// The original formulae looks like + /// The original formulae looks like /// \frac{\sum_{i = 1}^{n_x}{(x_i - \bar{x}) ^ 2} + \sum_{i = 1}^{n_y}{(y_i - \bar{y}) ^ 2}}{n_x + n_y - 2} /// But we made some mathematical transformations not to store original sequences. /// Also we dropped sqrt, because later it will be squared later. @@ -150,7 +158,8 @@ struct AggregateFunctionStudentTTestData final const Float64 t = getTStatisticSquared(); auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); - Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + int unused; + Float64 denominator = std::exp(lgammal_r(v / 2, &unused) + lgammal_r(0.5, &unused) - lgammal_r(v / 2 + 0.5, &unused)); return numenator / denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp index cd0599729b9..c8711c257f8 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp @@ -28,8 +28,8 @@ namespace void registerAggregateFunctionTimeSeriesGroupSum(AggregateFunctionFactory & factory) { - factory.registerFunction("timeSeriesGroupSum", createAggregateFunctionTimeSeriesGroupSum, AggregateFunctionFactory::CaseInsensitive); - factory.registerFunction("timeSeriesGroupRateSum", createAggregateFunctionTimeSeriesGroupSum, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("timeSeriesGroupSum", createAggregateFunctionTimeSeriesGroupSum); + factory.registerFunction("timeSeriesGroupRateSum", createAggregateFunctionTimeSeriesGroupSum); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 483c99dde9b..0dcb125305d 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -44,6 +44,6 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 175e0171606..b598f25162e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -18,11 +18,20 @@ #include + namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } +#if defined(OS_DARWIN) +extern "C" +{ + double lgammal_r(double x, int * signgamp); +} +#endif + + namespace DB { @@ -159,9 +168,10 @@ struct AggregateFunctionWelchTTestData final { const Float64 v = getDegreesOfFreedom(); const Float64 t = getTStatisticSquared(); - auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; + auto f = [&v] (double x) { return std::pow(x, v / 2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); - Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + int unused; + Float64 denominator = std::exp(lgammal_r(v / 2, &unused) + lgammal_r(0.5, &unused) - lgammal_r(v / 2 + 0.5, &unused)); return numenator / denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index 872c70c2b98..1e9f2782d95 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -58,7 +58,7 @@ AggregateFunctionPtr createAggregateFunctionWindowFunnel(const std::string & nam void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory & factory) { - factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel); } } diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index 3097070c651..eae24c1f3e9 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -39,8 +39,8 @@ namespace ErrorCodes namespace detail { -const size_t DEFAULT_SAMPLE_COUNT = 8192; -const auto MAX_SKIP_DEGREE = sizeof(UInt32) * 8; + const size_t DEFAULT_MAX_SAMPLE_SIZE = 8192; + const auto MAX_SKIP_DEGREE = sizeof(UInt32) * 8; } /// What if there is not a single value - throw an exception, or return 0 or NaN in the case of double? @@ -50,6 +50,7 @@ enum class ReservoirSamplerDeterministicOnEmpty RETURN_NAN_OR_ZERO, }; + template class ReservoirSamplerDeterministic @@ -60,8 +61,8 @@ class ReservoirSamplerDeterministic } public: - ReservoirSamplerDeterministic(const size_t sample_count_ = DEFAULT_SAMPLE_COUNT) - : sample_count{sample_count_} + ReservoirSamplerDeterministic(const size_t max_sample_size_ = detail::DEFAULT_MAX_SAMPLE_SIZE) + : max_sample_size{max_sample_size_} { } @@ -131,8 +132,8 @@ public: void merge(const ReservoirSamplerDeterministic & b) { - if (sample_count != b.sample_count) - throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different sample_count"); + if (max_sample_size != b.max_sample_size) + throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different max sample size"); sorted = false; if (b.skip_degree > skip_degree) @@ -150,11 +151,16 @@ public: void read(DB::ReadBuffer & buf) { - DB::readIntBinary(sample_count, buf); + size_t size = 0; + DB::readIntBinary(size, buf); DB::readIntBinary(total_values, buf); - samples.resize(std::min(total_values, sample_count)); - for (size_t i = 0; i < samples.size(); ++i) + /// Compatibility with old versions. + if (size > total_values) + size = total_values; + + samples.resize(size); + for (size_t i = 0; i < size; ++i) DB::readPODBinary(samples[i], buf); sorted = false; @@ -162,10 +168,11 @@ public: void write(DB::WriteBuffer & buf) const { - DB::writeIntBinary(sample_count, buf); + size_t size = samples.size(); + DB::writeIntBinary(size, buf); DB::writeIntBinary(total_values, buf); - for (size_t i = 0; i < std::min(sample_count, total_values); ++i) + for (size_t i = 0; i < size; ++i) DB::writePODBinary(samples[i], buf); } @@ -174,18 +181,19 @@ private: using Element = std::pair; using Array = DB::PODArray; - size_t sample_count; - size_t total_values{}; - bool sorted{}; + const size_t max_sample_size; /// Maximum amount of stored values. + size_t total_values = 0; /// How many values were inserted (regardless if they remain in sample or not). + bool sorted = false; Array samples; - UInt8 skip_degree{}; + UInt8 skip_degree = 0; /// The number N determining that we save only one per 2^N elements in average. void insertImpl(const T & v, const UInt32 hash) { - /// @todo why + 1? I don't quite recall - while (samples.size() + 1 >= sample_count) + /// Make a room for plus one element. + while (samples.size() >= max_sample_size) { - if (++skip_degree > detail::MAX_SKIP_DEGREE) + ++skip_degree; + if (skip_degree > detail::MAX_SKIP_DEGREE) throw DB::Exception{"skip_degree exceeds maximum value", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED}; thinOut(); } @@ -195,35 +203,17 @@ private: void thinOut() { - auto size = samples.size(); - for (size_t i = 0; i < size;) - { - if (!good(samples[i].second)) - { - /// swap current element with the last one - std::swap(samples[size - 1], samples[i]); - --size; - } - else - ++i; - } - - if (size != samples.size()) - { - samples.resize(size); - sorted = false; - } + samples.resize(std::distance(samples.begin(), + std::remove_if(samples.begin(), samples.end(), [this](const auto & elem){ return !good(elem.second); }))); + sorted = false; } void sortIfNeeded() { if (sorted) return; + std::sort(samples.begin(), samples.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; }); sorted = true; - std::sort(samples.begin(), samples.end(), [] (const std::pair & lhs, const std::pair & rhs) - { - return lhs.first < rhs.first; - }); } template diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index b7fb22aeb0e..550a44a23a2 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -6,6 +6,8 @@ #include #include +#include + #if defined(MEMORY_SANITIZER) #include #endif diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index bdbc941c1e7..51248a598af 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -344,7 +344,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Shift all NULL values to the end. for (const auto & [first, last] : equal_ranges) { - /// Current interval is righter than limit. + /// Current interval is righter than limit. if (limit && first > limit) break; diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index 99facea6055..99e134675f6 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -82,7 +82,7 @@ public: * @see DB::ColumnUnique * * The most common example uses https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ columns. - * Consider data type @e LC(String). The inner type here is @e String which is more or less a contigous memory + * Consider data type @e LC(String). The inner type here is @e String which is more or less a contiguous memory * region, so it can be easily represented as a @e StringRef. So we pass that ref to this function and get its * index in the dictionary, which can be used to operate with the indices column. */ diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index ef0b82666dd..3d6a2d6f99c 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -54,6 +54,7 @@ M(LocalThread, "Number of threads in local thread pools. Should be similar to GlobalThreadActive.") \ M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ M(DistributedFilesToInsert, "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.") \ + M(TablesToDropQueueSize, "Number of dropped tables, that are waiting for background data removal.") \ namespace CurrentMetrics { diff --git a/src/Common/FileSyncGuard.h b/src/Common/FileSyncGuard.h index 6451f6ebf36..486b02d0f24 100644 --- a/src/Common/FileSyncGuard.h +++ b/src/Common/FileSyncGuard.h @@ -5,15 +5,15 @@ namespace DB { -/// Helper class, that recieves file descriptor and does fsync for it in destructor. +/// Helper class, that receives file descriptor and does fsync for it in destructor. /// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end. /// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version. /// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496 class FileSyncGuard { public: - /// NOTE: If you have already opened descriptor, it's preffered to use - /// this constructor instead of construnctor with path. + /// NOTE: If you have already opened descriptor, it's preferred to use + /// this constructor instead of constructor with path. FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {} FileSyncGuard(const DiskPtr & disk_, const String & path) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 21116e9d432..e527e97d608 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -234,13 +234,13 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::is_same_v ? CurrentMetrics::GlobalThreadActive : CurrentMetrics::LocalThreadActive); job(); - /// job should be reseted before decrementing scheduled_jobs to + /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; } catch (...) { - /// job should be reseted before decrementing scheduled_jobs to + /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index fc5318b11fb..d10d5981d57 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -152,7 +152,7 @@ void TraceCollector::run() if (trace_log) { // time and time_in_microseconds are both being constructed from the same timespec so that the - // times will be equal upto the precision of a second. + // times will be equal up to the precision of a second. struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index abb8158781b..f5c57781eef 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1288,13 +1288,13 @@ void ZooKeeper::receiveEvent() response->removeRootPath(root_path); } - /// Instead of setting the watch in sendEvent, set it in receiveEvent becuase need to check the response. + /// Instead of setting the watch in sendEvent, set it in receiveEvent because need to check the response. /// The watch shouldn't be set if the node does not exist and it will never exist like sequential ephemeral nodes. /// By using getData() instead of exists(), a watch won't be set if the node doesn't exist. if (request_info.watch) { bool add_watch = false; - /// 3 indicates the ZooKeeperExistsRequest. + /// 3 indicates the ZooKeeperExistsRequest. // For exists, we set the watch on both node exist and nonexist case. // For other case like getData, we only set the watch when node exists. if (request_info.request->getOpNum() == 3) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index be2f697e1b3..7a6b605d015 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -185,9 +185,9 @@ void CompressedReadBufferBase::decompress(char * to, size_t size_decompressed, s } else { - throw Exception("Data compressed with different methods, given method byte " + throw Exception("Data compressed with different methods, given method byte 0x" + getHexUIntLowercase(method) - + ", previous method byte " + + ", previous method byte 0x" + getHexUIntLowercase(codec->getMethodByte()), ErrorCodes::CANNOT_DECOMPRESS); } diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index fe6a5b2dacd..46d7d7dfcc4 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -87,7 +87,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr else throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - /// Default codec replaced with current default codec which may dependend on different + /// Default codec replaced with current default codec which may depend on different /// settings (and properties of data) in runtime. CompressionCodecPtr result_codec; if (codec_family_name == DEFAULT_CODEC_NAME) diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index baf6e9b2b86..3746753df8b 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -26,7 +26,7 @@ void ICompressionCodec::setCodecDescription(const String & codec_name, const AST std::shared_ptr result = std::make_shared(); result->name = "CODEC"; - /// Special case for codec Multiple, which doens't have name. It's just list + /// Special case for codec Multiple, which doesn't have name. It's just list /// of other codecs. if (codec_name.empty()) { diff --git a/src/Core/Defines.h b/src/Core/Defines.h index ba3d37242fa..4d7d8e08ac3 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -67,11 +67,14 @@ /// Minimum revision supporting SettingsBinaryFormat::STRINGS. #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 +/// Minimum revision supporting OpenTelemetry +#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY 54442 + /// Mininum revision supporting interserver secret. #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 /// Version of ClickHouse TCP protocol. Increment it manually when you change the protocol. -#define DBMS_TCP_PROTOCOL_VERSION 54441 +#define DBMS_TCP_PROTOCOL_VERSION 54442 /// The boundary on which the blocks for asynchronous file operations should be aligned. #define DEFAULT_AIO_FILE_BLOCK_SIZE 4096 diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1179c0eb46b..c09c4b3b034 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -705,7 +705,7 @@ namespace MySQLReplication break; } default: - throw ReplicationError("Position update with unsupport event", ErrorCodes::LOGICAL_ERROR); + throw ReplicationError("Position update with unsupported event", ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b94883ca871..4e95d1ab395 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -222,6 +222,7 @@ class IColumn; M(UInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \ + M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ \ \ /** Limits during query execution are part of the settings. \ @@ -389,7 +390,7 @@ class IColumn; M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ M(Bool, allow_experimental_database_materialize_mysql, false, "Allow to create database with Engine=MaterializeMySQL(...).", 0) \ M(Bool, system_events_show_zero_values, false, "Include all metrics, even with zero values", 0) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ diff --git a/src/DataStreams/SquashingTransform.cpp b/src/DataStreams/SquashingTransform.cpp index c57e2351230..1f6ca8a7306 100644 --- a/src/DataStreams/SquashingTransform.cpp +++ b/src/DataStreams/SquashingTransform.cpp @@ -27,7 +27,7 @@ Block SquashingTransform::add(const Block & input_block) /* * To minimize copying, accept two types of argument: const reference for output - * stream, and rvalue reference for input stream, and decide whether to copy + * stream, and rvalue reference for input stream, and decide whether to copy * inside this function. This allows us not to copy Block unless we absolutely * have to. */ diff --git a/src/DataTypes/NumberTraits.h b/src/DataTypes/NumberTraits.h index 603449150db..77bd2101f05 100644 --- a/src/DataTypes/NumberTraits.h +++ b/src/DataTypes/NumberTraits.h @@ -29,7 +29,7 @@ constexpr size_t min(size_t x, size_t y) } /// @note There's no auto scale to larger big integer, only for integral ones. -/// It's cause of (U)Int64 backward compatibilty and very big performance penalties. +/// It's cause of (U)Int64 backward compatibility and very big performance penalties. constexpr size_t nextSize(size_t size) { if (size < 8) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index a9dbae8ec92..4fcd9f12276 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -116,7 +116,7 @@ void DatabaseAtomic::dropTable(const Context &, const String & table_name, bool } tryRemoveSymlink(table_name); /// Remove the inner table (if any) to avoid deadlock - /// (due to attemp to execute DROP from the worker thread) + /// (due to attempt to execute DROP from the worker thread) if (auto * mv = dynamic_cast(table.get())) mv->dropInnerTable(no_delay); /// Notify DatabaseCatalog that table was dropped. It will remove table data in background. @@ -261,21 +261,29 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora { DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); + bool locked_uuid = false; try { std::unique_lock lock{mutex}; if (query.database != database_name) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed to `{}`, cannot create table in `{}`", database_name, query.database); + /// Do some checks before renaming file from .tmp to .sql not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); - renameNoReplace(table_metadata_tmp_path, table_metadata_path); + /// We will get en exception if some table with the same UUID exists (even if it's detached table or table from another database) + DatabaseCatalog::instance().addUUIDMapping(query.uuid); + locked_uuid = true; + /// It throws if `table_metadata_path` already exists (it's possible if table was detached) + renameNoReplace(table_metadata_tmp_path, table_metadata_path); /// Commit point (a sort of) attachTableUnlocked(query.table, table, lock); /// Should never throw table_name_to_path.emplace(query.table, table_data_path); } catch (...) { Poco::File(table_metadata_tmp_path).remove(); + if (locked_uuid) + DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; } tryCreateSymlink(query.table, table_data_path); diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 81414902a33..0119f17f843 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -329,10 +329,4 @@ const StoragePtr & DatabaseLazyIterator::table() const return current_storage; } -void DatabaseLazyIterator::reset() -{ - if (current_storage) - current_storage.reset(); -} - } diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 58e5e465eef..2d091297c91 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -22,6 +22,10 @@ public: String getEngineName() const override { return "Lazy"; } + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + void loadStoredObjects( Context & context, bool has_force_restore_data_flag, bool force_attach) override; @@ -122,7 +126,6 @@ public: bool isValid() const override; const String & name() const override; const StoragePtr & table() const override; - void reset() override; private: const DatabaseLazy & database; diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 5eacb846d52..357acb32371 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -53,6 +53,9 @@ void DatabaseMemory::dropTable( } table->is_dropped = true; create_queries.erase(table_name); + UUID table_uuid = table->getStorageID().uuid; + if (table_uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().removeUUIDMappingFinally(table_uuid); } ASTPtr DatabaseMemory::getCreateDatabaseQuery() const diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp index ed85028d04d..6c5173c986f 100644 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ b/src/Databases/DatabaseWithDictionaries.cpp @@ -223,6 +223,10 @@ void DatabaseWithDictionaries::removeDictionary(const Context &, const String & attachDictionary(dictionary_name, attach_info); throw; } + + UUID dict_uuid = attach_info.create_query->as()->uuid; + if (dict_uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().removeUUIDMappingFinally(dict_uuid); } DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const FilterByNameFunction & filter_by_dictionary_name) diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 9b744259406..fadec5fe7a9 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -44,8 +44,6 @@ public: /// (a database with support for lazy tables loading /// - it maintains a list of tables but tables are loaded lazily). virtual const StoragePtr & table() const = 0; - /// Reset reference counter to the StoragePtr. - virtual void reset() = 0; virtual ~IDatabaseTablesIterator() = default; @@ -95,8 +93,6 @@ public: const String & name() const override { return it->first; } const StoragePtr & table() const override { return it->second; } - - void reset() override { it->second.reset(); } }; /// Copies list of dictionaries and iterates through such snapshot. @@ -151,6 +147,10 @@ public: /// Get name of database engine. virtual String getEngineName() const = 0; + virtual bool canContainMergeTreeTables() const { return true; } + + virtual bool canContainDistributedTables() const { return true; } + /// Load a set of existing tables. /// You can call only once, right after the object is created. virtual void loadStoredObjects(Context & /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} diff --git a/src/Databases/MySQL/ConnectionMySQLSettings.h b/src/Databases/MySQL/ConnectionMySQLSettings.h index 90279f846a4..ce2773307c5 100644 --- a/src/Databases/MySQL/ConnectionMySQLSettings.h +++ b/src/Databases/MySQL/ConnectionMySQLSettings.h @@ -11,7 +11,7 @@ class Context; class ASTStorage; #define LIST_OF_CONNECTION_MYSQL_SETTINGS(M) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ /// Settings that should not change after the creation of a database. #define APPLY_FOR_IMMUTABLE_CONNECTION_MYSQL_SETTINGS(M) \ diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.h b/src/Databases/MySQL/DatabaseConnectionMySQL.h index 7bf5e8c1d88..d8694e71db2 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.h +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.h @@ -42,6 +42,12 @@ public: String getEngineName() const override { return "MySQL"; } + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + + bool shouldBeEmptyOnDetach() const override { return false; } + bool empty() const override; DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override; diff --git a/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h b/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h index 5a0ec242c2f..86a5cbf8206 100644 --- a/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h +++ b/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h @@ -28,11 +28,6 @@ public: return tables.emplace_back(storage); } - void reset() override - { - tables.clear(); - } - UUID uuid() const override { return nested_iterator->uuid(); } DatabaseMaterializeTablesIterator(DatabaseTablesIteratorPtr nested_iterator_, DatabaseMaterializeMySQL * database_) diff --git a/src/Databases/MySQL/MaterializeMySQLSettings.cpp b/src/Databases/MySQL/MaterializeMySQLSettings.cpp index 609ce011f91..a8672bf488e 100644 --- a/src/Databases/MySQL/MaterializeMySQLSettings.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSettings.cpp @@ -8,7 +8,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -25,9 +24,8 @@ void MaterializeMySQLSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for database " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for database " + storage_def.engine->name); + throw; } } else diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 63df025d2b1..43ff42956cd 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -326,7 +326,7 @@ struct DecimalBinaryOperation } private: - /// there's implicit type convertion here + /// there's implicit type conversion here static NativeResultType apply(NativeResultType a, NativeResultType b) { if constexpr (can_overflow && check_overflow) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 10c4a27e509..68d8b41407d 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -577,7 +577,7 @@ private: auto input_value = input_column->getDataAt(r); if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) { - // empty plaintext results in empty ciphertext + tag, means there should be atleast tag_size bytes. + // empty plaintext results in empty ciphertext + tag, means there should be at least tag_size bytes. if (input_value.size < tag_size) throw Exception(fmt::format("Encrypted data is too short: only {} bytes, " "should contain at least {} bytes of a tag.", diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 3f38614f584..df962800385 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -68,6 +68,8 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); @@ -90,6 +92,8 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index fa9c363aff0..70e8904cfc1 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -569,7 +569,7 @@ template <> inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { UUID tmp; - readText(tmp, rb); + readUUIDText(tmp, rb); x = tmp; } @@ -603,6 +603,17 @@ inline bool tryParseImpl(DataTypeDateTime::FieldType & x, Read return true; } +template <> +inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) +{ + UUID tmp; + if (!tryReadUUIDText(tmp, rb)) + return false; + + x = tmp; + return true; +} + /** Throw exception with verbose message when string value is not parsed completely. */ @@ -1755,6 +1766,7 @@ struct NameToDecimal32OrZero { static constexpr auto name = "toDecimal32OrZero"; struct NameToDecimal64OrZero { static constexpr auto name = "toDecimal64OrZero"; }; struct NameToDecimal128OrZero { static constexpr auto name = "toDecimal128OrZero"; }; struct NameToDecimal256OrZero { static constexpr auto name = "toDecimal256OrZero"; }; +struct NameToUUIDOrZero { static constexpr auto name = "toUUIDOrZero"; }; using FunctionToUInt8OrZero = FunctionConvertFromString; using FunctionToUInt16OrZero = FunctionConvertFromString; @@ -1776,6 +1788,7 @@ using FunctionToDecimal32OrZero = FunctionConvertFromString, NameToDecimal64OrZero, ConvertFromStringExceptionMode::Zero>; using FunctionToDecimal128OrZero = FunctionConvertFromString, NameToDecimal128OrZero, ConvertFromStringExceptionMode::Zero>; using FunctionToDecimal256OrZero = FunctionConvertFromString, NameToDecimal256OrZero, ConvertFromStringExceptionMode::Zero>; +using FunctionToUUIDOrZero = FunctionConvertFromString; struct NameToUInt8OrNull { static constexpr auto name = "toUInt8OrNull"; }; struct NameToUInt16OrNull { static constexpr auto name = "toUInt16OrNull"; }; @@ -1797,6 +1810,7 @@ struct NameToDecimal32OrNull { static constexpr auto name = "toDecimal32OrNull"; struct NameToDecimal64OrNull { static constexpr auto name = "toDecimal64OrNull"; }; struct NameToDecimal128OrNull { static constexpr auto name = "toDecimal128OrNull"; }; struct NameToDecimal256OrNull { static constexpr auto name = "toDecimal256OrNull"; }; +struct NameToUUIDOrNull { static constexpr auto name = "toUUIDOrNull"; }; using FunctionToUInt8OrNull = FunctionConvertFromString; using FunctionToUInt16OrNull = FunctionConvertFromString; @@ -1818,6 +1832,7 @@ using FunctionToDecimal32OrNull = FunctionConvertFromString, NameToDecimal64OrNull, ConvertFromStringExceptionMode::Null>; using FunctionToDecimal128OrNull = FunctionConvertFromString, NameToDecimal128OrNull, ConvertFromStringExceptionMode::Null>; using FunctionToDecimal256OrNull = FunctionConvertFromString, NameToDecimal256OrNull, ConvertFromStringExceptionMode::Null>; +using FunctionToUUIDOrNull = FunctionConvertFromString; struct NameParseDateTimeBestEffort { static constexpr auto name = "parseDateTimeBestEffort"; }; struct NameParseDateTimeBestEffortUS { static constexpr auto name = "parseDateTimeBestEffortUS"; }; diff --git a/src/Functions/formatReadableTimeDelta.cpp b/src/Functions/formatReadableTimeDelta.cpp index e55829b1bad..2b574f672d3 100644 --- a/src/Functions/formatReadableTimeDelta.cpp +++ b/src/Functions/formatReadableTimeDelta.cpp @@ -131,7 +131,7 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - /// Virtual call is Ok (neglible comparing to the rest of calculations). + /// Virtual call is Ok (negligible comparing to the rest of calculations). Float64 value = arguments[0].column->getFloat64(i); bool is_negative = value < 0; diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index 11c413757c6..323c9951a96 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -22,7 +22,7 @@ namespace { /// Returns 1 if and Decimal value has more digits then it's Precision allow, 0 otherwise. -/// Precision could be set as second argument or omitted. If ommited function uses Decimal presicion of the first argument. +/// Precision could be set as second argument or omitted. If omitted function uses Decimal precision of the first argument. class FunctionIsDecimalOverflow : public IFunction { public: diff --git a/src/Functions/lgamma.cpp b/src/Functions/lgamma.cpp index 51b3dfd97df..e4da0d8dfbd 100644 --- a/src/Functions/lgamma.cpp +++ b/src/Functions/lgamma.cpp @@ -4,7 +4,6 @@ #if defined(OS_DARWIN) extern "C" { - /// Is defined in libglibc-compatibility.a double lgamma_r(double x, int * signgamp); } #endif diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 90a56af3c34..d79328889f1 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -619,9 +619,11 @@ inline bool tryReadDateText(DayNum & date, ReadBuffer & buf) return readDateTextImpl(date, buf); } - -inline void readUUIDText(UUID & uuid, ReadBuffer & buf) +template +inline ReturnType readUUIDTextImpl(UUID & uuid, ReadBuffer & buf) { + static constexpr bool throw_exception = std::is_same_v; + char s[36]; size_t size = buf.read(s, 32); @@ -634,21 +636,49 @@ inline void readUUIDText(UUID & uuid, ReadBuffer & buf) if (size != 36) { s[size] = 0; - throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + + if constexpr (throw_exception) + { + throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + } + else + { + return ReturnType(false); + } } parseUUID(reinterpret_cast(s), std::reverse_iterator(reinterpret_cast(&uuid) + 16)); } else parseUUIDWithoutSeparator(reinterpret_cast(s), std::reverse_iterator(reinterpret_cast(&uuid) + 16)); + + return ReturnType(true); } else { s[size] = 0; - throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + + if constexpr (throw_exception) + { + throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + } + else + { + return ReturnType(false); + } } } +inline void readUUIDText(UUID & uuid, ReadBuffer & buf) +{ + return readUUIDTextImpl(uuid, buf); +} + +inline bool tryReadUUIDText(UUID & uuid, ReadBuffer & buf) +{ + return readUUIDTextImpl(uuid, buf); +} + template inline T parse(const char * data, size_t size); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 7572968c2c6..22c81d1d365 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -744,12 +744,12 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { function_builder = FunctionFactory::instance().get(node.name, data.context); } - catch (DB::Exception & e) + catch (Exception & e) { auto hints = AggregateFunctionFactory::instance().getHints(node.name); if (!hints.empty()) e.addMessage("Or unknown aggregate function " + node.name + ". Maybe you meant: " + toString(hints)); - e.rethrow(); + throw; } Names argument_names; diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index feb2036a0d6..e1a9a820ebb 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -233,8 +233,8 @@ void AsynchronousMetrics::update() for (const auto & db : databases) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 71567a424c5..37eb403ddab 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -59,6 +59,26 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) writeVarUInt(client_version_patch, out); } + + if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) + { + if (opentelemetry_trace_id) + { + // Have OpenTelemetry header. + writeBinary(uint8_t(1), out); + // No point writing these numbers with variable length, because they + // are random and will probably require the full length anyway. + writeBinary(opentelemetry_trace_id, out); + writeBinary(opentelemetry_span_id, out); + writeBinary(opentelemetry_tracestate, out); + writeBinary(opentelemetry_trace_flags, out); + } + else + { + // Don't have OpenTelemetry header. + writeBinary(uint8_t(0), out); + } + } } @@ -112,6 +132,19 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) else client_version_patch = client_tcp_protocol_version; } + + if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) + { + uint8_t have_trace_id = 0; + readBinary(have_trace_id, in); + if (have_trace_id) + { + readBinary(opentelemetry_trace_id, in); + readBinary(opentelemetry_span_id, in); + readBinary(opentelemetry_tracestate, in); + readBinary(opentelemetry_trace_flags, in); + } + } } @@ -122,6 +155,74 @@ void ClientInfo::setInitialQuery() client_name = (DBMS_NAME " ") + client_name; } +bool ClientInfo::parseTraceparentHeader(const std::string & traceparent, + std::string & error) +{ + uint8_t version = -1; + uint64_t trace_id_high = 0; + uint64_t trace_id_low = 0; + uint64_t trace_parent = 0; + uint8_t trace_flags = 0; + + // Version 00, which is the only one we can parse, is fixed width. Use this + // fact for an additional sanity check. + const int expected_length = 2 + 1 + 32 + 1 + 16 + 1 + 2; + if (traceparent.length() != expected_length) + { + error = fmt::format("unexpected length {}, expected {}", + traceparent.length(), expected_length); + return false; + } + + // clang-tidy doesn't like sscanf: + // error: 'sscanf' used to convert a string to an unsigned integer value, + // but function will not report conversion errors; consider using 'strtoul' + // instead [cert-err34-c,-warnings-as-errors] + // There is no other ready solution, and hand-rolling a more complicated + // parser for an HTTP header in C++ sounds like RCE. + // NOLINTNEXTLINE(cert-err34-c) + int result = sscanf(&traceparent[0], + "%2" SCNx8 "-%16" SCNx64 "%16" SCNx64 "-%16" SCNx64 "-%2" SCNx8, + &version, &trace_id_high, &trace_id_low, &trace_parent, &trace_flags); + + if (result == EOF) + { + error = "EOF"; + return false; + } + + // We read uint128 as two uint64, so 5 parts and not 4. + if (result != 5) + { + error = fmt::format("could only read {} parts instead of the expected 5", + result); + return false; + } + + if (version != 0) + { + error = fmt::format("unexpected version {}, expected 00", version); + return false; + } + + opentelemetry_trace_id = static_cast<__uint128_t>(trace_id_high) << 64 + | trace_id_low; + opentelemetry_span_id = trace_parent; + opentelemetry_trace_flags = trace_flags; + return true; +} + + +std::string ClientInfo::composeTraceparentHeader() const +{ + // This span is a parent for its children, so we specify this span_id as a + // parent id. + return fmt::format("00-{:032x}-{:016x}-{:02x}", opentelemetry_trace_id, + opentelemetry_span_id, + // This cast is needed because fmt is being weird and complaining that + // "mixing character types is not allowed". + static_cast(opentelemetry_trace_flags)); +} void ClientInfo::fillOSUserHostNameAndVersionInfo() { diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 704f1913b89..2edf47684d3 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -58,6 +59,17 @@ public: String initial_query_id; Poco::Net::SocketAddress initial_address; + // OpenTelemetry trace information. + __uint128_t opentelemetry_trace_id = 0; + // The span id we get the in the incoming client info becomes our parent span + // id, and the span id we send becomes downstream parent span id. + UInt64 opentelemetry_span_id = 0; + UInt64 opentelemetry_parent_span_id = 0; + // The incoming tracestate header and the trace flags, we just pass them downstream. + // They are described at https://www.w3.org/TR/trace-context/ + String opentelemetry_tracestate; + UInt8 opentelemetry_trace_flags = 0; + /// All below are parameters related to initial query. Interface interface = Interface::TCP; @@ -90,6 +102,16 @@ public: /// Initialize parameters on client initiating query. void setInitialQuery(); + // Parse/compose OpenTelemetry traceparent header. + // Note that these functions use span_id field, not parent_span_id, same as + // in native protocol. The incoming traceparent corresponds to the upstream + // trace span, and the outgoing traceparent corresponds to our current span. + // We use the same ClientInfo structure first for incoming span, and then + // for our span: when we switch, we use old span_id as parent_span_id, and + // generate a new span_id (currently this happens in Context::setQueryId()). + bool parseTraceparentHeader(const std::string & traceparent, std::string & error); + std::string composeTraceparentHeader() const; + private: void fillOSUserHostNameAndVersionInfo(); }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 39d2df1882e..595f889ecd2 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1096,28 +1096,53 @@ void Context::setCurrentDatabase(const String & name) void Context::setCurrentQueryId(const String & query_id) { - String query_id_to_set = query_id; + /// Generate random UUID, but using lower quality RNG, + /// because Poco::UUIDGenerator::generateRandom method is using /dev/random, that is very expensive. + /// NOTE: Actually we don't need to use UUIDs for query identifiers. + /// We could use any suitable string instead. + union + { + char bytes[16]; + struct + { + UInt64 a; + UInt64 b; + } words; + __uint128_t uuid; + } random; + random.words.a = thread_local_rng(); //-V656 + random.words.b = thread_local_rng(); //-V656 + + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY + && client_info.opentelemetry_trace_id == 0) + { + // If this is an initial query without any parent OpenTelemetry trace, we + // might start the trace ourselves, with some configurable probability. + std::bernoulli_distribution should_start_trace{ + settings.opentelemetry_start_trace_probability}; + + if (should_start_trace(thread_local_rng)) + { + // Use the randomly generated default query id as the new trace id. + client_info.opentelemetry_trace_id = random.uuid; + client_info.opentelemetry_parent_span_id = 0; + client_info.opentelemetry_span_id = thread_local_rng(); + // Mark this trace as sampled in the flags. + client_info.opentelemetry_trace_flags = 1; + } + } + else + { + // The incoming request has an OpenTelemtry trace context. Its span id + // becomes our parent span id. + client_info.opentelemetry_parent_span_id = client_info.opentelemetry_span_id; + client_info.opentelemetry_span_id = thread_local_rng(); + } + + String query_id_to_set = query_id; if (query_id_to_set.empty()) /// If the user did not submit his query_id, then we generate it ourselves. { - /// Generate random UUID, but using lower quality RNG, - /// because Poco::UUIDGenerator::generateRandom method is using /dev/random, that is very expensive. - /// NOTE: Actually we don't need to use UUIDs for query identifiers. - /// We could use any suitable string instead. - - union - { - char bytes[16]; - struct - { - UInt64 a; - UInt64 b; - } words; - } random; - - random.words.a = thread_local_rng(); //-V656 - random.words.b = thread_local_rng(); //-V656 - /// Use protected constructor. struct QueryUUID : Poco::UUID { @@ -1741,6 +1766,17 @@ std::shared_ptr Context::getAsynchronousMetricLog() } +std::shared_ptr Context::getOpenTelemetrySpanLog() +{ + auto lock = getLock(); + + if (!shared->system_logs) + return {}; + + return shared->system_logs->opentelemetry_span_log; +} + + CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const { auto lock = getLock(); @@ -2004,10 +2040,15 @@ void Context::reloadConfig() const void Context::shutdown() { - for (auto & [disk_name, disk] : getDisksMap()) + // Disk selector might not be initialized if there was some error during + // its initialization. Don't try to initialize it again on shutdown. + if (shared->merge_tree_disk_selector) { - LOG_INFO(shared->log, "Shutdown disk {}", disk_name); - disk->shutdown(); + for (auto & [disk_name, disk] : getDisksMap()) + { + LOG_INFO(shared->log, "Shutdown disk {}", disk_name); + disk->shutdown(); + } } shared->shutdown(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 331883ee690..f8b9f747992 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -80,6 +80,7 @@ class TextLog; class TraceLog; class MetricLog; class AsynchronousMetricLog; +class OpenTelemetrySpanLog; struct MergeTreeSettings; class StorageS3Settings; class IDatabase; @@ -543,6 +544,7 @@ public: std::shared_ptr getTextLog(); std::shared_ptr getMetricLog(); std::shared_ptr getAsynchronousMetricLog(); + std::shared_ptr getOpenTelemetrySpanLog(); /// Returns an object used to log operations with parts if it possible. /// Provide table name to make required checks. diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 32d0e25bde5..fc7f5c2f765 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -1246,7 +1246,6 @@ public: size_t num_unfinished_hosts = waiting_hosts.size() - num_hosts_finished; size_t num_active_hosts = current_active_hosts.size(); - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Watching task {} is executing longer than distributed_ddl_task_timeout (={}) seconds. " "There are {} unfinished hosts ({} of them are currently active), they are going to execute the query in background", diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index cfe1046aceb..906863f3f44 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -13,9 +13,16 @@ #include #include #include +#include #include + +namespace CurrentMetrics +{ + extern const Metric TablesToDropQueueSize; +} + namespace DB { @@ -155,7 +162,17 @@ void DatabaseCatalog::shutdownImpl() tables_marked_dropped.clear(); std::lock_guard lock(databases_mutex); - assert(std::find_if_not(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { return elem.map.empty(); }) == uuid_map.end()); + assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) + { + /// Ensure that all UUID mappings are emtpy (i.e. all mappings contain nullptr instead of a pointer to storage) + const auto & not_empty_mapping = [] (const auto & mapping) + { + auto & table = mapping.second.second; + return table; + }; + auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); + return it != elem.map.end(); + }) == uuid_map.end()); databases.clear(); db_uuid_map.clear(); view_dependencies.clear(); @@ -411,36 +428,76 @@ DatabasePtr DatabaseCatalog::getSystemDatabase() const return getDatabase(SYSTEM_DATABASE); } -void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) +void DatabaseCatalog::addUUIDMapping(const UUID & uuid) +{ + addUUIDMapping(uuid, nullptr, nullptr); +} + +void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + assert((database && table) || (!database && !table)); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; - auto [_, inserted] = map_part.map.try_emplace(uuid, std::move(database), std::move(table)); + auto [it, inserted] = map_part.map.try_emplace(uuid, database, table); + if (inserted) + return; + + auto & prev_database = it->second.first; + auto & prev_table = it->second.second; + assert((prev_database && prev_table) || (!prev_database && !prev_table)); + + if (!prev_table && table) + { + /// It's empty mapping, it was created to "lock" UUID and prevent collision. Just update it. + prev_database = database; + prev_table = table; + return; + } + + /// We are trying to replace existing mapping (prev_table != nullptr), it's logical error + if (table) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", toString(uuid)); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR - if (!inserted) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " already exists", ErrorCodes::TABLE_ALREADY_EXISTS); + throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " + "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); } void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + if (it == map_part.map.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); + it->second = {}; +} + +void DatabaseCatalog::removeUUIDMappingFinally(const UUID & uuid) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; if (!map_part.map.erase(uuid)) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); } void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + assert(database && table); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; auto it = map_part.map.find(uuid); if (it == map_part.map.end()) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); - it->second = std::make_pair(std::move(database), std::move(table)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); + auto & prev_database = it->second.first; + auto & prev_table = it->second.second; + assert(prev_database && prev_table); + prev_database = std::move(database); + prev_table = std::move(table); } std::unique_ptr DatabaseCatalog::database_catalog; @@ -631,6 +688,8 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() dropped_metadata.emplace(std::move(full_path), std::move(dropped_id)); } + LOG_INFO(log, "Found {} partially dropped tables. Will load them and retry removal.", dropped_metadata.size()); + ThreadPool pool; for (const auto & elem : dropped_metadata) { @@ -695,6 +754,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr LOG_WARNING(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); } + addUUIDMapping(table_id.uuid); drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); } @@ -704,6 +764,8 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr else tables_marked_dropped.push_back({table_id, table, dropped_metadata_path, drop_time}); tables_marked_dropped_ids.insert(table_id.uuid); + CurrentMetrics::add(CurrentMetrics::TablesToDropQueueSize, 1); + /// If list of dropped tables was empty, start a drop task if (drop_task && tables_marked_dropped.size() == 1) (*drop_task)->schedule(); @@ -732,6 +794,10 @@ void DatabaseCatalog::dropTableDataTask() LOG_INFO(log, "Will try drop {}", table.table_id.getNameForLogs()); tables_marked_dropped.erase(it); } + else + { + LOG_TRACE(log, "Not found any suitable tables to drop, still have {} tables in drop queue", tables_marked_dropped.size()); + } need_reschedule = !tables_marked_dropped.empty(); } catch (...) @@ -770,7 +836,7 @@ void DatabaseCatalog::dropTableDataTask() (*drop_task)->scheduleAfter(reschedule_time_ms); } -void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const +void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) { if (table.table) { @@ -789,6 +855,9 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); Poco::File(table.metadata_path).remove(); + + removeUUIDMappingFinally(table.table_id.uuid); + CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1); } String DatabaseCatalog::getPathForUUID(const UUID & uuid) @@ -826,6 +895,8 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) { if (uuid == UUIDHelpers::Nil) return; + + LOG_DEBUG(log, "Waiting for table {} to be finally dropped", toString(uuid)); std::unique_lock lock{tables_marked_dropped_mutex}; wait_table_finally_dropped.wait(lock, [&]() { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index c6f50117564..d26307a3bc3 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -165,12 +165,21 @@ public: void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where); /// If table has UUID, addUUIDMapping(...) must be called when table attached to some database - /// and removeUUIDMapping(...) must be called when it detached. + /// removeUUIDMapping(...) must be called when it detached, + /// and removeUUIDMappingFinally(...) must be called when table is dropped and its data removed from disk. /// Such tables can be accessed by persistent UUID instead of database and table name. - void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + void addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table); void removeUUIDMapping(const UUID & uuid); + void removeUUIDMappingFinally(const UUID & uuid); /// For moving table between databases void updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + /// This method adds empty mapping (with database and storage equal to nullptr). + /// It's required to "lock" some UUIDs and protect us from collision. + /// Collisions of random 122-bit integers are very unlikely to happen, + /// but we allow to explicitly specify UUID in CREATE query (in particular for testing). + /// If some UUID was already added and we are trying to add it again, + /// this method will throw an exception. + void addUUIDMapping(const UUID & uuid); static String getPathForUUID(const UUID & uuid); @@ -222,7 +231,7 @@ private: void loadMarkedAsDroppedTables(); void dropTableDataTask(); - void dropTableFinally(const TableMarkedAsDropped & table) const; + void dropTableFinally(const TableMarkedAsDropped & table); static constexpr size_t reschedule_time_ms = 100; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 05647aef9f6..762ad6ae575 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -22,6 +22,15 @@ # include "config_core.h" #endif +#include + +#if defined(MEMORY_SANITIZER) + #include +#endif + +#if defined(ADDRESS_SANITIZER) + #include +#endif namespace ProfileEvents { @@ -624,6 +633,22 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } catch (Exception & e) { +#if defined(MEMORY_SANITIZER) + const auto & msg = e.message(); + if (__msan_test_shadow(msg.data(), msg.size()) != -1) + { + LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (msan): {}", e.getStackTraceString()); + } +#endif + +#if defined(ADDRESS_SANITIZER) + const auto & msg = e.message(); + if (__asan_region_is_poisoned(const_cast(msg.data()), msg.size())) + { + LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (asan): {}", e.getStackTraceString()); + } +#endif + e.addMessage(fmt::format("while executing '{}'", action.toString())); throw; } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index dcef36de175..e1713c7cbbb 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int DICTIONARIES_WAS_NOT_LOADED; } @@ -1404,7 +1405,29 @@ void ExternalLoader::checkLoaded(const ExternalLoader::LoadResult & result, if (result.status == ExternalLoader::Status::LOADING) throw Exception(type_name + " '" + result.name + "' is still loading", ErrorCodes::BAD_ARGUMENTS); if (result.exception) - std::rethrow_exception(result.exception); + { + // Exception is shared for multiple threads. + // Don't just rethrow it, because sharing the same exception object + // between multiple threads can lead to weird effects if they decide to + // modify it, for example, by adding some error context. + try + { + std::rethrow_exception(result.exception); + } + catch (const Poco::Exception & e) + { + /// This will create a copy for Poco::Exception and DB::Exception + e.rethrow(); + } + catch (...) + { + throw DB::Exception(ErrorCodes::DICTIONARIES_WAS_NOT_LOADED, + "Failed to load dictionary '{}': {}", + result.name, + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); + } + } if (result.status == ExternalLoader::Status::NOT_EXIST) throw Exception(type_name + " '" + result.name + "' not found", ErrorCodes::BAD_ARGUMENTS); if (result.status == ExternalLoader::Status::NOT_LOADED) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index c4ebe596649..8bf8675b15d 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -52,13 +52,37 @@ BlockIO InterpreterDropQuery::execute() return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); } else if (!drop.database.empty()) - return executeToDatabase(drop.database, drop.kind, drop.if_exists, drop.no_delay); + return executeToDatabase(drop); else throw Exception("Nothing to drop, both names are empty", ErrorCodes::LOGICAL_ERROR); } +void InterpreterDropQuery::waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait) +{ + if (uuid_to_wait == UUIDHelpers::Nil) + return; + + if (query.kind == ASTDropQuery::Kind::Drop) + DatabaseCatalog::instance().waitTableFinallyDropped(uuid_to_wait); + else if (query.kind == ASTDropQuery::Kind::Detach) + { + if (auto * atomic = typeid_cast(db.get())) + atomic->waitDetachedTableNotInUse(uuid_to_wait); + } +} + BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) +{ + DatabasePtr database; + UUID table_to_wait_on = UUIDHelpers::Nil; + auto res = executeToTableImpl(query, database, table_to_wait_on); + if (query.no_delay) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_to_wait_on); + return res; +} + +BlockIO InterpreterDropQuery::executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait) { /// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard auto table_id = StorageID(query); @@ -125,19 +149,9 @@ BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) database->dropTable(context, table_id.table_name, query.no_delay); } - } - table.reset(); - ddl_guard = {}; - if (query.no_delay) - { - if (query.kind == ASTDropQuery::Kind::Drop) - DatabaseCatalog::instance().waitTableFinallyDropped(table_id.uuid); - else if (query.kind == ASTDropQuery::Kind::Detach) - { - if (auto * atomic = typeid_cast(database.get())) - atomic->waitDetachedTableNotInUse(table_id.uuid); - } + db = database; + uuid_to_wait = table_id.uuid; } return {}; @@ -223,19 +237,48 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, } -BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay) +BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) { + DatabasePtr database; + std::vector tables_to_wait; + BlockIO res; + try + { + res = executeToDatabaseImpl(query, database, tables_to_wait); + } + catch (...) + { + if (query.no_delay) + { + for (const auto table_uuid : tables_to_wait) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); + } + throw; + } + + if (query.no_delay) + { + for (const auto table_uuid : tables_to_wait) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); + } + return res; +} + +BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait) +{ + const auto & database_name = query.database; auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); - if (auto database = tryGetDatabase(database_name, if_exists)) + database = tryGetDatabase(database_name, query.if_exists); + if (database) { - if (kind == ASTDropQuery::Kind::Truncate) + if (query.kind == ASTDropQuery::Kind::Truncate) { throw Exception("Unable to truncate database", ErrorCodes::SYNTAX_ERROR); } - else if (kind == ASTDropQuery::Kind::Detach || kind == ASTDropQuery::Kind::Drop) + else if (query.kind == ASTDropQuery::Kind::Detach || query.kind == ASTDropQuery::Kind::Drop) { - bool drop = kind == ASTDropQuery::Kind::Drop; + bool drop = query.kind == ASTDropQuery::Kind::Drop; context.checkAccess(AccessType::DROP_DATABASE, database_name); if (database->shouldBeEmptyOnDetach()) @@ -246,21 +289,22 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS for (auto iterator = database->getDictionariesIterator(); iterator->isValid(); iterator->next()) { String current_dictionary = iterator->name(); - executeToDictionary(database_name, current_dictionary, kind, false, false, false); + executeToDictionary(database_name, current_dictionary, query.kind, false, false, false); } - ASTDropQuery query; - query.kind = kind; - query.if_exists = true; - query.database = database_name; - query.no_delay = no_delay; + ASTDropQuery query_for_table; + query_for_table.kind = query.kind; + query_for_table.if_exists = true; + query_for_table.database = database_name; + query_for_table.no_delay = query.no_delay; for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { - /// Reset reference counter of the StoragePtr to allow synchronous drop. - iterator->reset(); - query.table = iterator->name(); - executeToTable(query); + DatabasePtr db; + UUID table_to_wait = UUIDHelpers::Nil; + query_for_table.table = iterator->name(); + executeToTableImpl(query_for_table, db, table_to_wait); + uuids_to_wait.push_back(table_to_wait); } } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 0e1fd47b079..fe5362985de 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -29,9 +29,13 @@ private: ASTPtr query_ptr; Context & context; - BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay); + BlockIO executeToDatabase(const ASTDropQuery & query); + BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait); BlockIO executeToTable(const ASTDropQuery & query); + BlockIO executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait); + + static void waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait); BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index ef7fd840ac5..cb5db386f5a 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -50,7 +50,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() return rewritten_query.str(); } - /// SHOW CLUSTER/CLUSTERS + /// SHOW CLUSTER/CLUSTERS if (query.clusters) { std::stringstream rewritten_query; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 102a06e648a..e59d0f925e9 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -338,7 +339,8 @@ BlockIO InterpreterSystemQuery::execute() [&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(true); }, [&] () { if (auto text_log = context.getTextLog()) text_log->flush(true); }, [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(true); }, - [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); } + [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); }, + [&] () { if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); } ); break; case Type::STOP_LISTEN_QUERIES: diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 70916fe386d..245feae166d 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -56,7 +56,7 @@ static inline String resolveDatabase( } } - /// When USE other_database_name; CREATE TABLE table_name; + /// When USE other_database_name; CREATE TABLE table_name; /// context.getCurrentDatabase() is always return `default database` /// When USE replica_mysql_database; CREATE TABLE table_name; /// context.getCurrentDatabase() is always return replica_clickhouse_database diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp new file mode 100644 index 00000000000..6c22165546d --- /dev/null +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -0,0 +1,47 @@ +#include "OpenTelemetrySpanLog.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +Block OpenTelemetrySpanLogElement::createBlock() +{ + return { + {std::make_shared(), "trace_id"}, + {std::make_shared(), "span_id"}, + {std::make_shared(), "parent_span_id"}, + {std::make_shared(), "operation_name"}, + {std::make_shared(6), "start_time_us"}, + {std::make_shared(6), "finish_time_us"}, + {std::make_shared(), "finish_date"}, + {std::make_shared(std::make_shared()), + "attribute.names"}, + {std::make_shared(std::make_shared()), + "attribute.values"} + }; +} + +void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const +{ + size_t i = 0; + + columns[i++]->insert(UInt128(Int128(trace_id))); + columns[i++]->insert(span_id); + columns[i++]->insert(parent_span_id); + columns[i++]->insert(operation_name); + columns[i++]->insert(start_time_us); + columns[i++]->insert(finish_time_us); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); + columns[i++]->insert(attribute_names); + columns[i++]->insert(attribute_values); +} + +} + diff --git a/src/Interpreters/OpenTelemetrySpanLog.h b/src/Interpreters/OpenTelemetrySpanLog.h new file mode 100644 index 00000000000..fb382ee3177 --- /dev/null +++ b/src/Interpreters/OpenTelemetrySpanLog.h @@ -0,0 +1,39 @@ +#pragma once + +#include + +namespace DB +{ + +struct OpenTelemetrySpan +{ + __uint128_t trace_id; + UInt64 span_id; + UInt64 parent_span_id; + std::string operation_name; + UInt64 start_time_us; + UInt64 finish_time_us; + UInt64 duration_ns; + Array attribute_names; + Array attribute_values; + // I don't understand how Links work, namely, which direction should they + // point to, and how they are related with parent_span_id, so no Links for + // now. +}; + +struct OpenTelemetrySpanLogElement : public OpenTelemetrySpan +{ + static std::string name() { return "OpenTelemetrySpanLog"; } + static Block createBlock(); + void appendToBlock(MutableColumns & columns) const; +}; + +// OpenTelemetry standartizes some Log data as well, so it's not just +// OpenTelemetryLog to avoid confusion. +class OpenTelemetrySpanLog : public SystemLog +{ +public: + using SystemLog::SystemLog; +}; + +} diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index 0faa748303d..f486752e192 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -16,7 +17,7 @@ struct PreparedSetKey /// if left hand sides of the IN operators have different types). static PreparedSetKey forLiteral(const IAST & ast, DataTypes types_) { - /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, + /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, /// just converts LowCardinality to ordinary types. for (auto & type : types_) type = recursiveRemoveLowCardinality(type); diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index a78b756e291..1e396a0fed1 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,9 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi asynchronous_metric_log = createSystemLog( global_context, "system", "asynchronous_metric_log", config, "asynchronous_metric_log"); + opentelemetry_span_log = createSystemLog( + global_context, "system", "opentelemetry_span_log", config, + "opentelemetry_span_log"); if (query_log) logs.emplace_back(query_log.get()); @@ -104,6 +108,8 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi logs.emplace_back(metric_log.get()); if (asynchronous_metric_log) logs.emplace_back(asynchronous_metric_log.get()); + if (opentelemetry_span_log) + logs.emplace_back(opentelemetry_span_log.get()); try { diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 99a85405348..6c56565a152 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -71,6 +71,7 @@ class TraceLog; class CrashLog; class MetricLog; class AsynchronousMetricLog; +class OpenTelemetrySpanLog; class ISystemLog @@ -105,6 +106,8 @@ struct SystemLogs std::shared_ptr metric_log; /// Used to log all metrics. /// Metrics from system.asynchronous_metrics. std::shared_ptr asynchronous_metric_log; + /// OpenTelemetry trace spans. + std::shared_ptr opentelemetry_span_log; std::vector logs; }; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 2166c78ef7c..7f29cfc7e5c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -166,7 +166,7 @@ void ThreadStatus::initPerformanceCounters() memory_tracker.setDescription("(for thread)"); // query_start_time_{microseconds, nanoseconds} are all constructed from the same time point - // to ensure that they are all equal upto the precision of a second. + // to ensure that they are all equal up to the precision of a second. const auto now = std::chrono::system_clock::now(); query_start_time_nanoseconds = time_in_nanoseconds(now); diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 9d52e30cea4..61ca933dd53 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -644,8 +644,13 @@ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & sou optimizeInjectiveFunctionsInsideUniq(query, context); /// Eliminate min/max/any aggregators of functions of GROUP BY keys - if (settings.optimize_aggregators_of_group_by_keys) + if (settings.optimize_aggregators_of_group_by_keys + && !select_query->group_by_with_totals + && !select_query->group_by_with_rollup + && !select_query->group_by_with_cube) + { optimizeAggregateFunctionsOfGroupByKeys(select_query, query); + } /// Remove duplicate items from ORDER BY. optimizeDuplicatesInOrderBy(select_query); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 57c557c5658..a672b58633d 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -139,15 +140,26 @@ static void logQuery(const String & query, const Context & context, bool interna } else { - const auto & current_query_id = context.getClientInfo().current_query_id; - const auto & initial_query_id = context.getClientInfo().initial_query_id; - const auto & current_user = context.getClientInfo().current_user; + const auto & client_info = context.getClientInfo(); + + const auto & current_query_id = client_info.current_query_id; + const auto & initial_query_id = client_info.initial_query_id; + const auto & current_user = client_info.current_user; LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}) {}", - context.getClientInfo().current_address.toString(), - (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), + client_info.current_address.toString(), + (current_user != "default" ? ", user: " + current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), joinLines(query)); + + if (client_info.opentelemetry_trace_id) + { + LOG_TRACE(&Poco::Logger::get("executeQuery"), + "OpenTelemetry trace id {:x}, span id {}, parent span id {}", + client_info.opentelemetry_trace_id, + client_info.opentelemetry_span_id, + client_info.opentelemetry_parent_span_id); + } } } @@ -194,7 +206,7 @@ inline UInt64 time_in_seconds(std::chrono::time_point return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); } -static void onExceptionBeforeStart(const String & query_for_logging, Context & context, time_t current_time, UInt64 current_time_microseconds, ASTPtr ast) +static void onExceptionBeforeStart(const String & query_for_logging, Context & context, UInt64 current_time_us, ASTPtr ast) { /// Exception before the query execution. if (auto quota = context.getQuota()) @@ -209,11 +221,11 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c // all callers to onExceptionBeforeStart method construct the timespec for event_time and // event_time_microseconds from the same time point. So, it can be assumed that both of these - // times are equal upto the precision of a second. - elem.event_time = current_time; - elem.event_time_microseconds = current_time_microseconds; - elem.query_start_time = current_time; - elem.query_start_time_microseconds = current_time_microseconds; + // times are equal up to the precision of a second. + elem.event_time = current_time_us / 1000000; + elem.event_time_microseconds = current_time_us; + elem.query_start_time = current_time_us / 1000000; + elem.query_start_time_microseconds = current_time_us; elem.current_database = context.getCurrentDatabase(); elem.query = query_for_logging; @@ -233,6 +245,39 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c if (auto query_log = context.getQueryLog()) query_log->add(elem); + if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog(); + context.getClientInfo().opentelemetry_trace_id + && opentelemetry_span_log) + { + OpenTelemetrySpanLogElement span; + span.trace_id = context.getClientInfo().opentelemetry_trace_id; + span.span_id = context.getClientInfo().opentelemetry_span_id; + span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; + span.operation_name = "query"; + span.start_time_us = current_time_us; + span.finish_time_us = current_time_us; + span.duration_ns = 0; + + // keep values synchonized to type enum in QueryLogElement::createBlock + span.attribute_names.push_back("clickhouse.query_status"); + span.attribute_values.push_back("ExceptionBeforeStart"); + + span.attribute_names.push_back("db.statement"); + span.attribute_values.push_back(elem.query); + + span.attribute_names.push_back("clickhouse.query_id"); + span.attribute_values.push_back(elem.client_info.current_query_id); + + if (!context.getClientInfo().opentelemetry_tracestate.empty()) + { + span.attribute_names.push_back("clickhouse.tracestate"); + span.attribute_values.push_back( + context.getClientInfo().opentelemetry_tracestate); + } + + opentelemetry_span_log->add(span); + } + ProfileEvents::increment(ProfileEvents::FailedQuery); if (ast) @@ -266,12 +311,7 @@ static std::tuple executeQueryImpl( bool has_query_tail, ReadBuffer * istr) { - // current_time and current_time_microseconds are both constructed from the same time point - // to ensure that both the times are equal upto the precision of a second. - const auto now = std::chrono::system_clock::now(); - - auto current_time = time_in_seconds(now); - auto current_time_microseconds = time_in_microseconds(now); + const auto current_time = std::chrono::system_clock::now(); /// If we already executing query and it requires to execute internal query, than /// don't replace thread context with given (it can be temporary). Otherwise, attach context to thread. @@ -322,7 +362,7 @@ static std::tuple executeQueryImpl( if (!internal) { - onExceptionBeforeStart(query_for_logging, context, current_time, current_time_microseconds, ast); + onExceptionBeforeStart(query_for_logging, context, time_in_microseconds(current_time), ast); } throw; @@ -494,10 +534,10 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::QUERY_START; - elem.event_time = current_time; - elem.event_time_microseconds = current_time_microseconds; - elem.query_start_time = current_time; - elem.query_start_time_microseconds = current_time_microseconds; + elem.event_time = time_in_seconds(current_time); + elem.event_time_microseconds = time_in_microseconds(current_time); + elem.query_start_time = time_in_seconds(current_time); + elem.query_start_time_microseconds = time_in_microseconds(current_time); elem.current_database = context.getCurrentDatabase(); elem.query = query_for_logging; @@ -568,9 +608,9 @@ static std::tuple executeQueryImpl( // construct event_time and event_time_microseconds using the same time point // so that the two times will always be equal up to a precision of a second. - const auto time_now = std::chrono::system_clock::now(); - elem.event_time = time_in_seconds(time_now); - elem.event_time_microseconds = time_in_microseconds(time_now); + const auto finish_time = std::chrono::system_clock::now(); + elem.event_time = time_in_seconds(finish_time); + elem.event_time_microseconds = time_in_microseconds(finish_time); status_info_to_query_log(elem, info, ast); auto progress_callback = context.getProgressCallback(); @@ -620,6 +660,38 @@ static std::tuple executeQueryImpl( if (auto query_log = context.getQueryLog()) query_log->add(elem); } + + if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog(); + context.getClientInfo().opentelemetry_trace_id + && opentelemetry_span_log) + { + OpenTelemetrySpanLogElement span; + span.trace_id = context.getClientInfo().opentelemetry_trace_id; + span.span_id = context.getClientInfo().opentelemetry_span_id; + span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; + span.operation_name = "query"; + span.start_time_us = elem.query_start_time_microseconds; + span.finish_time_us = time_in_microseconds(finish_time); + span.duration_ns = elapsed_seconds * 1000000000; + + // keep values synchonized to type enum in QueryLogElement::createBlock + span.attribute_names.push_back("clickhouse.query_status"); + span.attribute_values.push_back("QueryFinish"); + + span.attribute_names.push_back("db.statement"); + span.attribute_values.push_back(elem.query); + + span.attribute_names.push_back("clickhouse.query_id"); + span.attribute_values.push_back(elem.client_info.current_query_id); + if (!context.getClientInfo().opentelemetry_tracestate.empty()) + { + span.attribute_names.push_back("clickhouse.tracestate"); + span.attribute_values.push_back( + context.getClientInfo().opentelemetry_tracestate); + } + + opentelemetry_span_log->add(span); + } }; auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota), @@ -631,7 +703,7 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::EXCEPTION_WHILE_PROCESSING; // event_time and event_time_microseconds are being constructed from the same time point - // to ensure that both the times will be equal upto the precision of a second. + // to ensure that both the times will be equal up to the precision of a second. const auto time_now = std::chrono::system_clock::now(); elem.event_time = time_in_seconds(time_now); @@ -694,7 +766,7 @@ static std::tuple executeQueryImpl( if (query_for_logging.empty()) query_for_logging = prepareQueryForLogging(query, context); - onExceptionBeforeStart(query_for_logging, context, current_time, current_time_microseconds, ast); + onExceptionBeforeStart(query_for_logging, context, time_in_microseconds(current_time), ast); } throw; diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 4c0b64934c7..563b7374fdc 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -119,6 +119,7 @@ SRCS( MutationsInterpreter.cpp MySQL/InterpretersMySQLDDLQuery.cpp NullableUtils.cpp + OpenTelemetrySpanLog.cpp OptimizeIfChains.cpp OptimizeIfWithConstantConditionVisitor.cpp PartLog.cpp diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 32382b70bd7..11f4a77ee0e 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -927,7 +927,7 @@ void obfuscateQueries( } else { - /// Everyting else is kept as is. + /// Everything else is kept as is. result.write(token.begin, token.size()); } } diff --git a/src/Processors/IAccumulatingTransform.h b/src/Processors/IAccumulatingTransform.h index 3e77c798ad7..b51753199c3 100644 --- a/src/Processors/IAccumulatingTransform.h +++ b/src/Processors/IAccumulatingTransform.h @@ -36,7 +36,7 @@ public: Status prepare() override; void work() override; - /// Adds additional port fo totals. + /// Adds additional port for totals. /// If added, totals will have been ready by the first generate() call (in totals chunk). InputPort * addTotalsPort(); diff --git a/src/Processors/Transforms/ConvertingTransform.h b/src/Processors/Transforms/ConvertingTransform.h index b426a2ab525..4ae74457998 100644 --- a/src/Processors/Transforms/ConvertingTransform.h +++ b/src/Processors/Transforms/ConvertingTransform.h @@ -1,7 +1,9 @@ #pragma once + #include #include + namespace DB { @@ -46,7 +48,7 @@ private: /// How to construct result block. Position in source block, where to get each column. ColumnNumbers conversion; /// Do not check that constants are same. Use value from result_header. - /// This is needed in case run functions which are constant in query scope, + /// This is needed in case run functions which are constant in query scope, /// but may return different result being executed remotely, like `now64()` or `randConstant()`. /// In this case we replace constants from remote source to constatns from initiator. bool ignore_constant_values; diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 53483d6f7fb..5e03e1d178c 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -95,6 +95,7 @@ namespace ErrorCodes extern const int WRONG_PASSWORD; extern const int REQUIRED_PASSWORD; + extern const int BAD_REQUEST_PARAMETER; extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; } @@ -279,9 +280,7 @@ void HTTPHandler::processQuery( } } - std::string query_id = params.get("query_id", ""); context.setUser(user, password, request.clientAddress()); - context.setCurrentQueryId(query_id); if (!quota_key.empty()) context.setQuotaKey(quota_key); @@ -311,6 +310,31 @@ void HTTPHandler::processQuery( session->release(); }); + // Parse the OpenTelemetry traceparent header. + // Disable in Arcadia -- it interferes with the + // test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test. +#if !defined(ARCADIA_BUILD) + if (request.has("traceparent")) + { + std::string opentelemetry_traceparent = request.get("traceparent"); + std::string error; + if (!context.getClientInfo().parseTraceparentHeader( + opentelemetry_traceparent, error)) + { + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, + "Failed to parse OpenTelemetry traceparent header '{}': {}", + opentelemetry_traceparent, error); + } + + context.getClientInfo().opentelemetry_tracestate = request.get("tracestate", ""); + } +#endif + + // Set the query id supplied by the user, if any, and also update the + // OpenTelemetry fields. + context.setCurrentQueryId(params.get("query_id", + request.get("X-ClickHouse-Query-Id", ""))); + /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); CompressionMethod http_response_compression_method = CompressionMethod::None; diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index bc5436f00ee..1aa5c10afd7 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -43,8 +43,8 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request /// Iterate through all the replicated tables. for (const auto & db : databases) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index a37f88f9306..1b0cbc69b29 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -896,8 +896,6 @@ void TCPHandler::receiveQuery() state.is_empty = false; readStringBinary(state.query_id, *in); - query_context->setCurrentQueryId(state.query_id); - /// Client info ClientInfo & client_info = query_context->getClientInfo(); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) @@ -917,14 +915,6 @@ void TCPHandler::receiveQuery() /// Set fields, that are known apriori. client_info.interface = ClientInfo::Interface::TCP; - if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - { - /// 'Current' fields was set at receiveHello. - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; - } - /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS @@ -1001,11 +991,32 @@ void TCPHandler::receiveQuery() query_context->clampToSettingsConstraints(settings_changes); } query_context->applySettingsChanges(settings_changes); - const Settings & settings = query_context->getSettingsRef(); + + // Use the received query id, or generate a random default. It is convenient + // to also generate the default OpenTelemetry trace id at the same time, and + // set the trace parent. + // Why is this done here and not earlier: + // 1) ClientInfo might contain upstream trace id, so we decide whether to use + // the default ids after we have received the ClientInfo. + // 2) There is the opentelemetry_start_trace_probability setting that + // controls when we start a new trace. It can be changed via Native protocol, + // so we have to apply the changes first. + query_context->setCurrentQueryId(state.query_id); + + // Set parameters of initial query. + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + { + /// 'Current' fields was set at receiveHello. + client_info.initial_user = client_info.current_user; + client_info.initial_query_id = client_info.current_query_id; + client_info.initial_address = client_info.current_address; + } + /// Sync timeouts on client and server during current query to avoid dangling queries on server /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), /// because settings.send_timeout is client-side setting which has opposite meaning on the server side. /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) + const Settings & settings = query_context->getSettingsRef(); state.timeout_setter = std::make_unique(socket(), settings.receive_timeout, settings.send_timeout); } diff --git a/src/Storages/CMakeLists.txt b/src/Storages/CMakeLists.txt index ae47fba063a..deb1c9f6716 100644 --- a/src/Storages/CMakeLists.txt +++ b/src/Storages/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(MergeTree) add_subdirectory(System) if(ENABLE_TESTS) diff --git a/src/Storages/JoinSettings.cpp b/src/Storages/JoinSettings.cpp index 15637d67dea..8a2699746da 100644 --- a/src/Storages/JoinSettings.cpp +++ b/src/Storages/JoinSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void JoinSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/Kafka/KafkaSettings.cpp b/src/Storages/Kafka/KafkaSettings.cpp index 4d80419af2d..6ef74511d83 100644 --- a/src/Storages/Kafka/KafkaSettings.cpp +++ b/src/Storages/Kafka/KafkaSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void KafkaSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/MergeTree/CMakeLists.txt b/src/Storages/MergeTree/CMakeLists.txt new file mode 100644 index 00000000000..36cab0b3590 --- /dev/null +++ b/src/Storages/MergeTree/CMakeLists.txt @@ -0,0 +1,3 @@ +if(ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 319b486c2c6..ffc2dd62ce0 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -353,7 +353,7 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const return checksum->second.file_size; } -String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const +String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const { const auto & storage_columns = metadata_snapshot->getColumns().getAllPhysical(); auto alter_conversions = storage.getAlterConversionsForPart(shared_from_this()); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 202d9494247..21932ba445c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -145,7 +145,7 @@ public: /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const; + String getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const; bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 5b044622b36..ba6c2a3d462 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -68,7 +68,7 @@ MergeInfo MergeListElement::getInfo() const res.memory_usage = memory_tracker.get(); res.thread_id = thread_id; res.merge_type = toString(merge_type); - res.merge_algorithm = toString(merge_algorithm); + res.merge_algorithm = toString(merge_algorithm.load(std::memory_order_relaxed)); for (const auto & source_part_name : source_part_names) res.source_part_names.emplace_back(source_part_name); diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index c1166c55703..09c61250afd 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -92,7 +92,8 @@ struct MergeListElement : boost::noncopyable UInt64 thread_id; MergeType merge_type; - MergeAlgorithm merge_algorithm; + /// Detected after merge already started + std::atomic merge_algorithm; MergeListElement(const std::string & database, const std::string & table, const FutureMergedMutatedPart & future_part); diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 739dfedfde4..ad10a437b1e 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -95,7 +95,7 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetada */ if (!have_at_least_one_physical_column) { - const auto minimum_size_column_name = part->getColumnNameWithMinumumCompressedSize(metadata_snapshot); + const auto minimum_size_column_name = part->getColumnNameWithMinimumCompressedSize(metadata_snapshot); columns.push_back(minimum_size_column_name); /// correctly report added column injected_columns.insert(columns.back()); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 1bfd54ade85..055a5ec7b7a 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -298,7 +298,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( if (metadata_snapshot->hasAnyTTL() && merge_with_ttl_allowed && !ttl_merges_blocker.isCancelled()) { - /// TTL delete is prefered to recompression + /// TTL delete is preferred to recompression TTLDeleteMergeSelector delete_ttl_selector( next_delete_ttl_merge_times_by_partition, current_time, @@ -710,10 +710,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; size_t sum_compressed_bytes_upper_bound = merge_entry->total_size_bytes_compressed; - MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); - merge_entry->merge_algorithm = merge_alg; + MergeAlgorithm chosen_merge_algorithm = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); + merge_entry->merge_algorithm.store(chosen_merge_algorithm, std::memory_order_relaxed); - LOG_DEBUG(log, "Selected MergeAlgorithm: {}", toString(merge_alg)); + LOG_DEBUG(log, "Selected MergeAlgorithm: {}", toString(chosen_merge_algorithm)); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex /// (which is locked in data.getTotalActiveSizeInBytes()) @@ -728,7 +728,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor std::unique_ptr rows_sources_write_buf; std::optional column_sizes; - if (merge_alg == MergeAlgorithm::Vertical) + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { tmp_disk->createDirectories(new_part_tmp_path); rows_sources_file_path = new_part_tmp_path + "rows_sources"; @@ -818,7 +818,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ProcessorPtr merged_transform; /// If merge is vertical we cannot calculate it - bool blocks_are_granules_size = (merge_alg == MergeAlgorithm::Vertical); + bool blocks_are_granules_size = (chosen_merge_algorithm == MergeAlgorithm::Vertical); UInt64 merge_block_size = data_settings->merge_max_block_size; switch (data.merging_params.mode) @@ -917,7 +917,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { /// The same progress from merge_entry could be used for both algorithms (it should be more accurate) /// But now we are using inaccurate row-based estimation in Horizontal case for backward compatibility - Float64 progress = (merge_alg == MergeAlgorithm::Horizontal) + Float64 progress = (chosen_merge_algorithm == MergeAlgorithm::Horizontal) ? std::min(1., 1. * rows_written / sum_input_rows_upper_bound) : std::min(1., merge_entry->progress.load(std::memory_order_relaxed)); @@ -938,7 +938,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::DataPart::Checksums checksums_gathered_columns; /// Gather ordinary columns - if (merge_alg == MergeAlgorithm::Vertical) + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { size_t sum_input_rows_exact = merge_entry->rows_read; merge_entry->columns_written = merging_column_names.size(); @@ -1054,7 +1054,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ReadableSize(merge_entry->bytes_read_uncompressed / elapsed_seconds)); } - if (merge_alg != MergeAlgorithm::Vertical) + if (chosen_merge_algorithm != MergeAlgorithm::Vertical) to.writeSuffixAndFinalizePart(new_data_part, need_sync); else to.writeSuffixAndFinalizePart(new_data_part, need_sync, &storage_columns, &checksums_gathered_columns); diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h index f0837f98486..17239e2618a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h @@ -49,7 +49,7 @@ struct MergeTreeDataPartTTLInfos TTLInfoMap recompression_ttl; - /// Return smalles max recompression TTL value + /// Return the smallest max recompression TTL value time_t getMinimalMaxRecompressionTTL() const; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8c1dc845d26..44d0788901f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1517,7 +1517,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { /// In case when SELECT's predicate defines a single continuous interval of keys, /// we can use binary search algorithm to find the left and right endpoint key marks of such interval. - /// The returned value is the minumum range of marks, containing all keys for which KeyCondition holds + /// The returned value is the minimum range of marks, containing all keys for which KeyCondition holds LOG_TRACE(log, "Running binary search on index range for part {} ({} marks)", part->name, marks_count); diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 00580c8d8bb..15ff62e0aa6 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -11,9 +11,8 @@ namespace DB namespace ErrorCodes { - extern const int INVALID_CONFIG_PARAMETER; - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; + extern const int BAD_ARGUMENTS; } IMPLEMENT_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS) @@ -34,9 +33,8 @@ void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::U catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " in MergeTree config", ErrorCodes::INVALID_CONFIG_PARAMETER); - else - e.rethrow(); + e.addMessage("in MergeTree config"); + throw; } } @@ -51,9 +49,8 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 1d133f73a7b..4b5ae580257 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -64,7 +64,7 @@ void MergeTreeWriteAheadLog::addPart(const Block & block, const String & part_na min_block_number = std::min(min_block_number, part_info.min_block); max_block_number = std::max(max_block_number, part_info.max_block); - writeIntBinary(static_cast(0), *out); /// version + writeIntBinary(WAL_VERSION, *out); writeIntBinary(static_cast(ActionType::ADD_PART), *out); writeStringBinary(part_name, *out); block_out->write(block); @@ -80,7 +80,7 @@ void MergeTreeWriteAheadLog::dropPart(const String & part_name) { std::unique_lock lock(write_mutex); - writeIntBinary(static_cast(0), *out); + writeIntBinary(WAL_VERSION, *out); writeIntBinary(static_cast(ActionType::DROP_PART), *out); writeStringBinary(part_name, *out); out->next(); @@ -116,9 +116,13 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor try { + ActionMetadata metadata; + readIntBinary(version, *in); - if (version != 0) - throw Exception("Unknown WAL format version: " + toString(version), ErrorCodes::UNKNOWN_FORMAT_VERSION); + if (version > 0) + { + metadata.read(*in); + } readIntBinary(action_type, *in); readStringBinary(part_name, *in); @@ -233,4 +237,29 @@ MergeTreeWriteAheadLog::tryParseMinMaxBlockNumber(const String & filename) return std::make_pair(min_block, max_block); } +void MergeTreeWriteAheadLog::ActionMetadata::read(ReadBuffer & meta_in) +{ + readIntBinary(min_compatible_version, meta_in); + if (min_compatible_version > WAL_VERSION) + throw Exception("WAL metadata version " + toString(min_compatible_version) + + " is not compatible with this ClickHouse version", ErrorCodes::UNKNOWN_FORMAT_VERSION); + + size_t metadata_size; + readVarUInt(metadata_size, meta_in); + + UInt32 metadata_start = meta_in.offset(); + + /// For the future: read metadata here. + + + /// Skip extra fields if any. If min_compatible_version is lower than WAL_VERSION it means + /// that the fields are not critical for the correctness. + meta_in.ignore(metadata_size - (meta_in.offset() - metadata_start)); +} + +void MergeTreeWriteAheadLog::ActionMetadata::write(WriteBuffer & meta_out) const +{ + writeIntBinary(min_compatible_version, meta_out); + writeVarUInt(static_cast(0), meta_out); +} } diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h index 77c7c7e11e7..f4cf8ddc315 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h @@ -28,6 +28,19 @@ public: DROP_PART = 1, }; + struct ActionMetadata + { + /// The minimum version of WAL reader that can understand metadata written by current ClickHouse version. + /// This field must be increased when making backwards incompatible changes. + /// + /// The same approach can be used recursively inside metadata. + UInt8 min_compatible_version = 0; + + void write(WriteBuffer & meta_out) const; + void read(ReadBuffer & meta_in); + }; + + constexpr static UInt8 WAL_VERSION = 0; constexpr static auto WAL_FILE_NAME = "wal"; constexpr static auto WAL_FILE_EXTENSION = ".bin"; constexpr static auto DEFAULT_WAL_FILE_NAME = "wal.bin"; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 95358ecee97..8cfa798902a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -57,6 +57,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & return virtual_parts.getContainingPart(data_part->info) != data_part->name; } + bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { auto queue_path = replica_path + "/queue"; @@ -68,6 +69,9 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { std::lock_guard pull_logs_lock(pull_logs_to_queue_mutex); + /// Reset batch size on initialization to recover from possible errors of too large batch size. + current_multi_batch_size = 1; + String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); @@ -486,20 +490,21 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper { std::sort(log_entries.begin(), log_entries.end()); - /// ZK contains a limit on the number or total size of operations in a multi-request. - /// If the limit is exceeded, the connection is simply closed. - /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. - /// The average size of the node value in this case is less than 10 kilobytes. - static constexpr auto MAX_MULTI_OPS = 100; - - for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries; entry_idx += MAX_MULTI_OPS) + for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries;) { auto begin = log_entries.begin() + entry_idx; - auto end = entry_idx + MAX_MULTI_OPS >= log_entries.size() + auto end = entry_idx + current_multi_batch_size >= log_entries.size() ? log_entries.end() - : (begin + MAX_MULTI_OPS); + : (begin + current_multi_batch_size); auto last = end - 1; + /// Increment entry_idx before batch size increase (we copied at most current_multi_batch_size entries) + entry_idx += current_multi_batch_size; + + /// Increase the batch size exponentially, so it will saturate to MAX_MULTI_OPS. + if (current_multi_batch_size < MAX_MULTI_OPS) + current_multi_batch_size = std::min(MAX_MULTI_OPS, current_multi_batch_size * 2); + String last_entry = *last; if (!startsWith(last_entry, "log-")) throw Exception("Error in zookeeper data: unexpected node " + last_entry + " in " + zookeeper_path + "/log", diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index ead97579a4f..afbd59b0ce2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -260,6 +260,18 @@ private: }; using CurrentlyExecutingPtr = std::unique_ptr; + /// ZK contains a limit on the number or total size of operations in a multi-request. + /// If the limit is exceeded, the connection is simply closed. + /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. + /// The average size of the node value in this case is less than 10 kilobytes. + static constexpr size_t MAX_MULTI_OPS = 100; + + /// Very large queue entries may appear occasionally. + /// We cannot process MAX_MULTI_OPS at once because it will fail. + /// But we have to process more than one entry at once because otherwise lagged replicas keep up slowly. + /// Let's start with one entry per transaction and icrease it exponentially towards MAX_MULTI_OPS. + /// It will allow to make some progress before failing and remain operational even in extreme cases. + size_t current_multi_batch_size = 1; public: ReplicatedMergeTreeQueue(StorageReplicatedMergeTree & storage_); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h index 986253a2206..824ed73c171 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h @@ -36,7 +36,7 @@ private: Poco::Logger * log; std::atomic need_stop {false}; - // We need it besides `storage.is_readonly`, bacause `shutdown()` may be called many times, that way `storage.is_readonly` will not change. + // We need it besides `storage.is_readonly`, because `shutdown()` may be called many times, that way `storage.is_readonly` will not change. bool incr_readonly = false; /// The random data we wrote into `/replicas/me/is_active`. diff --git a/src/Storages/MergeTree/TTLMergeSelector.h b/src/Storages/MergeTree/TTLMergeSelector.h index c294687cdc5..710b85b3474 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.h +++ b/src/Storages/MergeTree/TTLMergeSelector.h @@ -84,7 +84,7 @@ public: time_t getTTLForPart(const IMergeSelector::Part & part) const override; /// Checks that part's codec is not already equal to required codec - /// according to recompression TTL. It doesn't make sence to assign such + /// according to recompression TTL. It doesn't make sense to assign such /// merge. bool isTTLAlreadySatisfied(const IMergeSelector::Part & part) const override; private: diff --git a/src/Storages/MergeTree/tests/CMakeLists.txt b/src/Storages/MergeTree/tests/CMakeLists.txt new file mode 100644 index 00000000000..777c75f191a --- /dev/null +++ b/src/Storages/MergeTree/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (wal_action_metadata wal_action_metadata.cpp) +target_link_libraries (wal_action_metadata PRIVATE dbms) diff --git a/src/Storages/MergeTree/tests/wal_action_metadata.cpp b/src/Storages/MergeTree/tests/wal_action_metadata.cpp new file mode 100644 index 00000000000..03c38c7a186 --- /dev/null +++ b/src/Storages/MergeTree/tests/wal_action_metadata.cpp @@ -0,0 +1,61 @@ +#include + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNKNOWN_FORMAT_VERSION; +} +} + +int main(int, char **) +{ + try + { + { + std::cout << "test: dummy test" << std::endl; + + DB::MergeTreeWriteAheadLog::ActionMetadata metadata_out; + DB::MemoryWriteBuffer buf{}; + + metadata_out.write(buf); + buf.finalize(); + + metadata_out.read(*buf.tryGetReadBuffer()); + } + + { + std::cout << "test: min compatibility" << std::endl; + + DB::MergeTreeWriteAheadLog::ActionMetadata metadata_out; + metadata_out.min_compatible_version = DB::MergeTreeWriteAheadLog::WAL_VERSION + 1; + DB::MemoryWriteBuffer buf{}; + + metadata_out.write(buf); + buf.finalize(); + + try + { + metadata_out.read(*buf.tryGetReadBuffer()); + } + catch (const DB::Exception & e) + { + if (e.code() != DB::ErrorCodes::UNKNOWN_FORMAT_VERSION) + { + std::cerr << "Expected UNKNOWN_FORMAT_VERSION exception but got: " + << e.what() << ", " << e.displayText() << std::endl; + } + } + } + } + catch (const DB::Exception & e) + { + std::cerr << e.what() << ", " << e.displayText() << std::endl; + return 1; + } + + return 0; +} diff --git a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp index c74081d8802..830c6224b9e 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp @@ -27,7 +27,7 @@ RabbitMQBlockInputStream::RabbitMQBlockInputStream( , non_virtual_header(metadata_snapshot->getSampleBlockNonMaterialized()) , sample_block(non_virtual_header) , virtual_header(metadata_snapshot->getSampleBlockForColumns( - {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id"}, + {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id", "_timestamp"}, storage.getVirtuals(), storage.getStorageID())) { for (const auto & column : virtual_header) @@ -158,6 +158,7 @@ Block RabbitMQBlockInputStream::readImpl() auto delivery_tag = buffer->getDeliveryTag(); auto redelivered = buffer->getRedelivered(); auto message_id = buffer->getMessageID(); + auto timestamp = buffer->getTimestamp(); buffer->updateAckTracker({delivery_tag, channel_id}); @@ -168,6 +169,7 @@ Block RabbitMQBlockInputStream::readImpl() virtual_columns[2]->insert(delivery_tag); virtual_columns[3]->insert(redelivered); virtual_columns[4]->insert(message_id); + virtual_columns[5]->insert(timestamp); } total_rows = total_rows + new_rows; diff --git a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h index f68b79275f6..5f2c2a62018 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h +++ b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h @@ -30,6 +30,7 @@ public: Block readImpl() override; void readSuffixImpl() override; + bool queueEmpty() const { return !buffer || buffer->queueEmpty(); } bool needChannelUpdate(); void updateChannel(); bool sendAck(); diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.cpp b/src/Storages/RabbitMQ/RabbitMQSettings.cpp index f956c520749..93495cdd8ae 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSettings.cpp @@ -9,7 +9,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -26,9 +25,8 @@ void RabbitMQSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 43a9d75d084..404ba27ccde 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -14,47 +14,27 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING; -} - ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, - ChannelPtr setup_channel_, HandlerPtr event_handler_, - const String & exchange_name_, + std::vector & queues_, size_t channel_id_base_, const String & channel_base_, - const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - bool hash_exchange_, - size_t num_queues_, - const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_) : ReadBuffer(nullptr, 0) , consumer_channel(std::move(consumer_channel_)) - , setup_channel(setup_channel_) , event_handler(event_handler_) - , exchange_name(exchange_name_) + , queues(queues_) , channel_base(channel_base_) , channel_id_base(channel_id_base_) - , queue_base(queue_base_) - , hash_exchange(hash_exchange_) - , num_queues(num_queues_) - , deadletter_exchange(deadletter_exchange_) , log(log_) , row_delimiter(row_delimiter_) - , queue_size(queue_size_) , stopped(stopped_) - , received(queue_size * num_queues) + , received(queue_size_) { - for (size_t queue_id = 0; queue_id < num_queues; ++queue_id) - bindQueue(queue_id); - setupChannel(); } @@ -65,67 +45,6 @@ ReadBufferFromRabbitMQConsumer::~ReadBufferFromRabbitMQConsumer() } -void ReadBufferFromRabbitMQConsumer::bindQueue(size_t queue_id) -{ - std::atomic binding_created = false; - - auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) - { - queues.emplace_back(queue_name); - LOG_DEBUG(log, "Queue {} is declared", queue_name); - - if (msgcount) - LOG_INFO(log, "Queue {} is non-empty. Non-consumed messaged will also be delivered", queue_name); - - /* Here we bind either to sharding exchange (consistent-hash) or to bridge exchange (fanout). All bindings to routing keys are - * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for - * fanout exchange it can be arbitrary - */ - setup_channel->bindQueue(exchange_name, queue_name, std::to_string(channel_id_base)) - .onSuccess([&] { binding_created = true; }) - .onError([&](const char * message) - { - throw Exception( - ErrorCodes::CANNOT_CREATE_RABBITMQ_QUEUE_BINDING, - "Failed to create queue binding with queue {} for exchange {}. Reason: {}", std::string(message), - queue_name, exchange_name); - }); - }; - - auto error_callback([&](const char * message) - { - /* This error is most likely a result of an attempt to declare queue with different settings if it was declared before. So for a - * given queue name either deadletter_exchange parameter changed or queue_size changed, i.e. table was declared with different - * max_block_size parameter. Solution: client should specify a different queue_base parameter or manually delete previously - * declared queues via any of the various cli tools. - */ - throw Exception("Failed to declare queue. Probably queue settings are conflicting: max_block_size, deadletter_exchange. Attempt \ - specifying differently those settings or use a different queue_base or manually delete previously declared queues, \ - which were declared with the same names. ERROR reason: " - + std::string(message), ErrorCodes::BAD_ARGUMENTS); - }); - - AMQP::Table queue_settings; - - queue_settings["x-max-length"] = queue_size; - queue_settings["x-overflow"] = "reject-publish"; - - if (!deadletter_exchange.empty()) - queue_settings["x-dead-letter-exchange"] = deadletter_exchange; - - /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one - * specific queue when its name is specified in queue_base setting - */ - const String queue_name = !hash_exchange ? queue_base : std::to_string(channel_id_base) + "_" + std::to_string(queue_id) + "_" + queue_base; - setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); - - while (!binding_created) - { - iterateEventLoop(); - } -} - - void ReadBufferFromRabbitMQConsumer::subscribe() { for (const auto & queue_name : queues) @@ -146,16 +65,15 @@ void ReadBufferFromRabbitMQConsumer::subscribe() if (row_delimiter != '\0') message_received += row_delimiter; - if (message.hasMessageID()) - received.push({message_received, message.messageID(), redelivered, AckTracker(delivery_tag, channel_id)}); - else - received.push({message_received, "", redelivered, AckTracker(delivery_tag, channel_id)}); + received.push({message_received, message.hasMessageID() ? message.messageID() : "", + message.hasTimestamp() ? message.timestamp() : 0, + redelivered, AckTracker(delivery_tag, channel_id)}); } }) .onError([&](const char * message) { /* End up here either if channel ends up in an error state (then there will be resubscription) or consume call error, which - * arises from queue settings mismatch or queue level error, which should not happen as noone else is supposed to touch them + * arises from queue settings mismatch or queue level error, which should not happen as no one else is supposed to touch them */ LOG_ERROR(log, "Consumer failed on channel {}. Reason: {}", channel_id, message); wait_subscription.store(false); @@ -173,16 +91,16 @@ bool ReadBufferFromRabbitMQConsumer::ackMessages() */ if (record_info.channel_id == channel_id && record_info.delivery_tag && record_info.delivery_tag > prev_tag) { - /// Commit all received messages with delivery tags from last commited to last inserted + /// Commit all received messages with delivery tags from last committed to last inserted if (!consumer_channel->ack(record_info.delivery_tag, AMQP::multiple)) { - LOG_ERROR(log, "Failed to commit messages with delivery tags from last commited to {} on channel {}", + LOG_ERROR(log, "Failed to commit messages with delivery tags from last committed to {} on channel {}", record_info.delivery_tag, channel_id); return false; } prev_tag = record_info.delivery_tag; - LOG_TRACE(log, "Consumer commited messages with deliveryTags up to {} on channel {}", record_info.delivery_tag, channel_id); + LOG_TRACE(log, "Consumer committed messages with deliveryTags up to {} on channel {}", record_info.delivery_tag, channel_id); } return true; diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 109770c77e9..476db3f5e94 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -24,17 +24,12 @@ class ReadBufferFromRabbitMQConsumer : public ReadBuffer public: ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, - ChannelPtr setup_channel_, HandlerPtr event_handler_, - const String & exchange_name_, + std::vector & queues_, size_t channel_id_base_, const String & channel_base_, - const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - bool hash_exchange_, - size_t num_queues_, - const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_); @@ -53,6 +48,7 @@ public: { String message; String message_id; + uint64_t timestamp; bool redelivered; AckTracker track; }; @@ -75,34 +71,26 @@ public: auto getDeliveryTag() const { return current.track.delivery_tag; } auto getRedelivered() const { return current.redelivered; } auto getMessageID() const { return current.message_id; } + auto getTimestamp() const { return current.timestamp; } private: bool nextImpl() override; - void bindQueue(size_t queue_id); void subscribe(); void iterateEventLoop(); ChannelPtr consumer_channel; - ChannelPtr setup_channel; HandlerPtr event_handler; - - const String exchange_name; + std::vector queues; const String channel_base; const size_t channel_id_base; - const String queue_base; - const bool hash_exchange; - const size_t num_queues; - const String deadletter_exchange; Poco::Logger * log; char row_delimiter; bool allowed = true; - uint32_t queue_size; const std::atomic & stopped; String channel_id; std::atomic channel_error = true, wait_subscription = false; - std::vector queues; ConcurrentBoundedQueue received; MessageData current; size_t subscribed = 0; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 6565afeb32d..9735c4d7fd3 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -38,8 +38,10 @@ namespace DB static const auto CONNECT_SLEEP = 200; static const auto RETRIES_MAX = 20; -static const auto HEARTBEAT_RESCHEDULE_MS = 3000; static const uint32_t QUEUE_SIZE = 100000; +static const auto MAX_FAILED_READ_ATTEMPTS = 10; +static const auto RESCHEDULE_MS = 500; +static const auto MAX_THREAD_WORK_DURATION_MS = 60000; namespace ErrorCodes { @@ -50,6 +52,7 @@ namespace ErrorCodes extern const int CANNOT_BIND_RABBITMQ_EXCHANGE; extern const int CANNOT_DECLARE_RABBITMQ_EXCHANGE; extern const int CANNOT_REMOVE_RABBITMQ_EXCHANGE; + extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING; } namespace ExchangeType @@ -122,9 +125,6 @@ StorageRabbitMQ::StorageRabbitMQ( streaming_task = global_context.getSchedulePool().createTask("RabbitMQStreamingTask", [this]{ streamingToViewsFunc(); }); streaming_task->deactivate(); - heartbeat_task = global_context.getSchedulePool().createTask("RabbitMQHeartbeatTask", [this]{ heartbeatFunc(); }); - heartbeat_task->deactivate(); - if (queue_base.empty()) { /* Make sure that local exchange name is unique for each table and is not the same as client's exchange name. It also needs to @@ -210,16 +210,6 @@ Context StorageRabbitMQ::addSettings(Context context) const } -void StorageRabbitMQ::heartbeatFunc() -{ - if (!stream_cancelled && event_handler->connectionRunning()) - { - connection->heartbeat(); - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); - } -} - - void StorageRabbitMQ::loopingFunc() { if (event_handler->connectionRunning()) @@ -396,17 +386,77 @@ void StorageRabbitMQ::bindExchange() } +void StorageRabbitMQ::bindQueue(size_t queue_id) +{ + std::atomic binding_created = false; + + auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) + { + queues.emplace_back(queue_name); + LOG_DEBUG(log, "Queue {} is declared", queue_name); + + if (msgcount) + LOG_INFO(log, "Queue {} is non-empty. Non-consumed messaged will also be delivered", queue_name); + + /* Here we bind either to sharding exchange (consistent-hash) or to bridge exchange (fanout). All bindings to routing keys are + * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for + * fanout exchange it can be arbitrary + */ + setup_channel->bindQueue(consumer_exchange, queue_name, std::to_string(queue_id)) + .onSuccess([&] { binding_created = true; }) + .onError([&](const char * message) + { + throw Exception( + ErrorCodes::CANNOT_CREATE_RABBITMQ_QUEUE_BINDING, + "Failed to create queue binding for exchange {}. Reason: {}", exchange_name, std::string(message)); + }); + }; + + auto error_callback([&](const char * message) + { + /* This error is most likely a result of an attempt to declare queue with different settings if it was declared before. So for a + * given queue name either deadletter_exchange parameter changed or queue_size changed, i.e. table was declared with different + * max_block_size parameter. Solution: client should specify a different queue_base parameter or manually delete previously + * declared queues via any of the various cli tools. + */ + throw Exception("Failed to declare queue. Probably queue settings are conflicting: max_block_size, deadletter_exchange. Attempt \ + specifying differently those settings or use a different queue_base or manually delete previously declared queues, \ + which were declared with the same names. ERROR reason: " + + std::string(message), ErrorCodes::BAD_ARGUMENTS); + }); + + AMQP::Table queue_settings; + + queue_settings["x-max-length"] = queue_size; + + if (!deadletter_exchange.empty()) + queue_settings["x-dead-letter-exchange"] = deadletter_exchange; + else + queue_settings["x-overflow"] = "reject-publish"; + + /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one + * specific queue when its name is specified in queue_base setting + */ + const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; + setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); + + while (!binding_created) + { + event_handler->iterateLoop(); + } +} + + bool StorageRabbitMQ::restoreConnection(bool reconnecting) { size_t cnt_retries = 0; if (reconnecting) { - deactivateTask(heartbeat_task, false, false); connection->close(); /// Connection might be unusable, but not closed /* Connection is not closed immediately (firstly, all pending operations are completed, and then - * an AMQP closing-handshake is performed). But cannot open a new connection untill previous one is properly closed + * an AMQP closing-handshake is performed). But cannot open a new connection until previous one is properly closed */ while (!connection->closed() && ++cnt_retries != RETRIES_MAX) event_handler->iterateLoop(); @@ -452,11 +502,11 @@ void StorageRabbitMQ::unbindExchange() */ std::call_once(flag, [&]() { - heartbeat_task->deactivate(); streaming_task->deactivate(); event_handler->updateLoopState(Loop::STOP); looping_task->deactivate(); + setup_channel = std::make_shared(connection.get()); setup_channel->removeExchange(bridge_exchange) .onSuccess([&]() { @@ -471,6 +521,8 @@ void StorageRabbitMQ::unbindExchange() { event_handler->iterateLoop(); } + + setup_channel->close(); }); } @@ -499,8 +551,6 @@ Pipe StorageRabbitMQ::read( deactivateTask(looping_task, false, true); update_channels = restoreConnection(true); - if (update_channels) - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); } Pipes pipes; @@ -521,7 +571,6 @@ Pipe StorageRabbitMQ::read( if (event_handler->loopRunning()) { deactivateTask(looping_task, false, true); - deactivateTask(heartbeat_task, false, false); } rabbit_stream->updateChannel(); @@ -552,6 +601,13 @@ void StorageRabbitMQ::startup() initExchange(); bindExchange(); + for (size_t i = 1; i <= num_queues; ++i) + { + bindQueue(i); + } + + setup_channel->close(); + for (size_t i = 0; i < num_consumers; ++i) { try @@ -568,7 +624,6 @@ void StorageRabbitMQ::startup() event_handler->updateLoopState(Loop::RUN); streaming_task->activateAndSchedule(); - heartbeat_task->activateAndSchedule(); } @@ -579,7 +634,6 @@ void StorageRabbitMQ::shutdown() deactivateTask(streaming_task, true, false); deactivateTask(looping_task, true, true); - deactivateTask(heartbeat_task, true, false); connection->close(); @@ -635,9 +689,8 @@ ConsumerBufferPtr StorageRabbitMQ::createReadBuffer() ChannelPtr consumer_channel = std::make_shared(connection.get()); return std::make_shared( - consumer_channel, setup_channel, event_handler, consumer_exchange, ++consumer_id, - unique_strbase, queue_base, log, row_delimiter, hash_exchange, num_queues, - deadletter_exchange, queue_size, stream_cancelled); + consumer_channel, event_handler, queues, ++consumer_id, + unique_strbase, log, row_delimiter, queue_size, stream_cancelled); } @@ -683,11 +736,14 @@ void StorageRabbitMQ::streamingToViewsFunc() try { auto table_id = getStorageID(); + // Check if at least one direct dependency is attached size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); if (dependencies_count) { + auto start_time = std::chrono::steady_clock::now(); + // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0) { @@ -696,8 +752,17 @@ void StorageRabbitMQ::streamingToViewsFunc() LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); - if (!streamToViews()) + if (streamToViews()) break; + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time); + if (duration.count() > MAX_THREAD_WORK_DURATION_MS) + { + event_handler->updateLoopState(Loop::STOP); + LOG_TRACE(log, "Reschedule streaming. Thread work duration limit exceeded."); + break; + } } } } @@ -708,7 +773,7 @@ void StorageRabbitMQ::streamingToViewsFunc() /// Wait for attached views if (!stream_cancelled) - streaming_task->schedule(); + streaming_task->scheduleAfter(RESCHEDULE_MS); } @@ -731,13 +796,6 @@ bool StorageRabbitMQ::streamToViews() auto column_names = block_io.out->getHeader().getNames(); auto sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); - /* event_handler->connectionRunning() does not guarantee that connnection is not closed in case loop was not running before, but - * need to anyway start the loop to activate error callbacks and update connection state, because even checking with - * connection->usable() will not give correct answer before callbacks are activated. - */ - if (!event_handler->loopRunning() && event_handler->connectionRunning()) - looping_task->activateAndSchedule(); - auto block_size = getMaxBlockSize(); // Create a stream for each consumer and join them in a union stream @@ -770,34 +828,45 @@ bool StorageRabbitMQ::streamToViews() in = streams[0]; std::atomic stub = {false}; + + if (!event_handler->loopRunning()) + { + event_handler->updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); + } + copyData(*in, *block_io.out, &stub); - /* Need to stop loop even if connection is ok, because sending ack() with loop running in another thread will lead to a lot of data - * races inside the library, but only in case any error occurs or connection is lost while ack is being sent + /* Note: sending ack() with loop running in another thread will lead to a lot of data races inside the library, but only in case + * error occurs or connection is lost while ack is being sent */ - if (event_handler->loopRunning()) - deactivateTask(looping_task, false, true); + deactivateTask(looping_task, false, true); + size_t queue_empty = 0; if (!event_handler->connectionRunning()) { - if (!stream_cancelled && restoreConnection(true)) + if (stream_cancelled) + return true; + + if (restoreConnection(true)) { for (auto & stream : streams) stream->as()->updateChannel(); } else { - /// Reschedule if unable to connect to rabbitmq or quit if cancelled - return false; + LOG_TRACE(log, "Reschedule streaming. Unable to restore connection."); + return true; } } else { - deactivateTask(heartbeat_task, false, false); - /// Commit for (auto & stream : streams) { + if (stream->as()->queueEmpty()) + ++queue_empty; + /* false is returned by the sendAck function in only two cases: * 1) if connection failed. In this case all channels will be closed and will be unable to send ack. Also ack is made based on * delivery tags, which are unique to channels, so if channels fail, those delivery tags will become invalid and there is @@ -828,19 +897,25 @@ bool StorageRabbitMQ::streamToViews() break; } } + + event_handler->iterateLoop(); } } - event_handler->updateLoopState(Loop::RUN); - looping_task->activateAndSchedule(); - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); /// It is also deactivated in restoreConnection(), so reschedule anyway + if ((queue_empty == num_created_consumers) && (++read_attempts == MAX_FAILED_READ_ATTEMPTS)) + { + connection->heartbeat(); + read_attempts = 0; + LOG_TRACE(log, "Reschedule streaming. Queues are empty."); + return true; + } + else + { + event_handler->updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); + } - // Check whether the limits were applied during query execution - bool limits_applied = false; - const BlockStreamProfileInfo & info = in->getProfileInfo(); - limits_applied = info.hasAppliedLimit(); - - return limits_applied; + return false; } @@ -907,7 +982,8 @@ NamesAndTypesList StorageRabbitMQ::getVirtuals() const {"_channel_id", std::make_shared()}, {"_delivery_tag", std::make_shared()}, {"_redelivered", std::make_shared()}, - {"_message_id", std::make_shared()} + {"_message_id", std::make_shared()}, + {"_timestamp", std::make_shared()} }; } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 8d9a20f9e34..d7891aed0a7 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -114,14 +114,15 @@ private: std::atomic wait_confirm = true; /// needed to break waiting for confirmations for producer std::atomic exchange_removed = false; ChannelPtr setup_channel; + std::vector queues; std::once_flag flag; /// remove exchange only once std::mutex task_mutex; BackgroundSchedulePool::TaskHolder streaming_task; - BackgroundSchedulePool::TaskHolder heartbeat_task; BackgroundSchedulePool::TaskHolder looping_task; std::atomic stream_cancelled{false}; + size_t read_attempts = 0; ConsumerBufferPtr createReadBuffer(); @@ -140,6 +141,7 @@ private: void initExchange(); void bindExchange(); + void bindQueue(size_t queue_id); bool restoreConnection(bool reconnecting); bool streamToViews(); diff --git a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h index a8e94070dbd..28fa5df8111 100644 --- a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h +++ b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h @@ -76,20 +76,20 @@ private: String channel_id; /* payloads.queue: - * - payloads are pushed to queue in countRow and poped by another thread in writingFunc, each payload gets into queue only once + * - payloads are pushed to queue in countRow and popped by another thread in writingFunc, each payload gets into queue only once * returned.queue: * - payloads are pushed to queue: * 1) inside channel->onError() callback if channel becomes unusable and the record of pending acknowledgements from server * is non-empty. * 2) inside removeRecord() if received nack() - negative acknowledgement from the server that message failed to be written * to disk or it was unable to reach the queue. - * - payloads are poped from the queue once republished + * - payloads are popped from the queue once republished */ ConcurrentBoundedQueue> payloads, returned; /* Counter of current delivery on a current channel. Delivery tags are scoped per channel. The server attaches a delivery tag for each * published message - a serial number of delivery on current channel. Delivery tag is a way of server to notify publisher if it was - * able or unable to process delivery, i.e. it sends back a responce with a corresponding delivery tag. + * able or unable to process delivery, i.e. it sends back a response with a corresponding delivery tag. */ UInt64 delivery_tag = 0; @@ -100,7 +100,7 @@ private: */ bool wait_all = true; - /* false: untill writeSuffix is called + /* false: until writeSuffix is called * true: means payloads.queue will not grow anymore */ std::atomic wait_num = 0; diff --git a/src/Storages/SetSettings.cpp b/src/Storages/SetSettings.cpp index f7ff1c446f2..baa3d231067 100644 --- a/src/Storages/SetSettings.cpp +++ b/src/Storages/SetSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void SetSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9046940b3f7..0c1561fca9b 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -80,7 +80,6 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int TOO_MANY_ROWS; extern const int UNABLE_TO_SKIP_UNUSED_SHARDS; - extern const int LOGICAL_ERROR; } namespace ActionLocks @@ -600,15 +599,22 @@ void StorageDistributed::shutdown() monitors_blocker.cancelForever(); std::lock_guard lock(cluster_nodes_mutex); + + LOG_DEBUG(log, "Joining background threads for async INSERT"); cluster_nodes_data.clear(); + LOG_DEBUG(log, "Background threads for async INSERT joined"); } void StorageDistributed::drop() { - // shutdown() should be already called - // and by the same reason we cannot use truncate() here, since - // cluster_nodes_data already cleaned - if (!cluster_nodes_data.empty()) - throw Exception("drop called before shutdown", ErrorCodes::LOGICAL_ERROR); + // Some INSERT in-between shutdown() and drop() can call + // requireDirectoryMonitor() again, so call shutdown() to clear them, but + // when the drop() (this function) executed none of INSERT is allowed in + // parallel. + // + // And second time shutdown() should be fast, since none of + // DirectoryMonitor should do anything, because ActionBlocker is canceled + // (in shutdown()). + shutdown(); // Distributed table w/o sharding_key does not allows INSERTs if (relative_data_path.empty()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 53f2ff14b3b..f64a53e3091 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3469,8 +3469,10 @@ void StorageReplicatedMergeTree::startup() { queue.initialize(getDataParts()); - data_parts_exchange_endpoint = std::make_shared(*this); - global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint); + InterserverIOEndpointPtr data_parts_exchange_ptr = std::make_shared(*this); + [[maybe_unused]] auto prev_ptr = std::atomic_exchange(&data_parts_exchange_endpoint, data_parts_exchange_ptr); + assert(prev_ptr == nullptr); + global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_ptr->getId(replica_path), data_parts_exchange_ptr); /// In this thread replica will be activated. restarting_thread.start(); @@ -3522,15 +3524,15 @@ void StorageReplicatedMergeTree::shutdown() } background_moves_executor.finish(); - if (data_parts_exchange_endpoint) + auto data_parts_exchange_ptr = std::atomic_exchange(&data_parts_exchange_endpoint, InterserverIOEndpointPtr{}); + if (data_parts_exchange_ptr) { - global_context.getInterserverIOHandler().removeEndpointIfExists(data_parts_exchange_endpoint->getId(replica_path)); + global_context.getInterserverIOHandler().removeEndpointIfExists(data_parts_exchange_ptr->getId(replica_path)); /// Ask all parts exchange handlers to finish asap. New ones will fail to start - data_parts_exchange_endpoint->blocker.cancelForever(); + data_parts_exchange_ptr->blocker.cancelForever(); /// Wait for all of them - std::unique_lock lock(data_parts_exchange_endpoint->rwlock); + std::unique_lock lock(data_parts_exchange_ptr->rwlock); } - data_parts_exchange_endpoint.reset(); /// We clear all old parts after stopping all background operations. It's /// important, because background operations can produce temporary parts @@ -5873,7 +5875,10 @@ ActionLock StorageReplicatedMergeTree::getActionLock(StorageActionBlockType acti return fetcher.blocker.cancel(); if (action_type == ActionLocks::PartsSend) - return data_parts_exchange_endpoint ? data_parts_exchange_endpoint->blocker.cancel() : ActionLock(); + { + auto data_parts_exchange_ptr = std::atomic_load(&data_parts_exchange_endpoint); + return data_parts_exchange_ptr ? data_parts_exchange_ptr->blocker.cancel() : ActionLock(); + } if (action_type == ActionLocks::ReplicationQueue) return queue.actions_blocker.cancel(); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index a6c8f1b39f8..55c16496ba5 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -67,6 +67,22 @@ namespace const CompressionMethod compression_method) : SourceWithProgress(sample_block), name(std::move(name_)) { + ReadWriteBufferFromHTTP::HTTPHeaderEntries header; + + // Propagate OpenTelemetry trace context, if any, downstream. + const auto & client_info = context.getClientInfo(); + if (client_info.opentelemetry_trace_id) + { + header.emplace_back("traceparent", + client_info.composeTraceparentHeader()); + + if (!client_info.opentelemetry_tracestate.empty()) + { + header.emplace_back("tracestate", + client_info.opentelemetry_tracestate); + } + } + read_buf = wrapReadBufferWithCompressionMethod( std::make_unique( uri, @@ -76,7 +92,7 @@ namespace context.getSettingsRef().max_http_get_redirects, Poco::Net::HTTPBasicCredentials{}, DBMS_DEFAULT_BUFFER_SIZE, - ReadWriteBufferFromHTTP::HTTPHeaderEntries{}, + header, context.getRemoteHostFilter()), compression_method); diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 2459be0ba71..39ccea64e26 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -38,8 +38,8 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, cons std::map> tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain distributed tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain distributed tables + if (!db.second->canContainDistributedTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index ffa789a4751..93bc16785b2 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -32,8 +32,8 @@ static StorageSystemGraphite::Configs getConfigs(const Context & context) for (const auto & db : databases) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 32f672b8401..f66f57ef5d1 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -44,8 +44,8 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex std::map> merge_tree_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index faa2ec0e1c3..d10346af89f 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -83,9 +83,9 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const MutableColumnPtr database_column_mut = ColumnString::create(); for (const auto & database : databases) { - /// Lazy database can not contain MergeTree tables - /// and it's unnecessary to load all tables of Lazy database just to filter all of them. - if (database.second->getEngineName() != "Lazy") + /// Checck if database can contain MergeTree tables, + /// if not it's unnecessary to load all tables of database just to filter all of them. + if (database.second->canContainMergeTreeTables()) database_column_mut->insert(database.first); } block_to_filter.insert(ColumnWithTypeAndName( diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 7ab6e939815..ab54d760873 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -74,8 +74,8 @@ Pipe StorageSystemReplicas::read( std::map> replicated_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index f04d8759507..9cd5e8b8ff3 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -55,8 +55,8 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const std::map> replicated_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 418b8ce0356..4e5f36a5f5d 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -333,7 +333,7 @@ "with_coverage": false } }, - "Functional stateless tests (release, polymorphic parts enabled)": { + "Functional stateless tests (release, wide parts enabled)": { "required_build_properties": { "compiler": "clang-11", "package_type": "deb", diff --git a/tests/clickhouse-test b/tests/clickhouse-test index d5736001d60..626d7ede814 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -180,7 +180,7 @@ def need_retry(stderr): def get_processlist(client_cmd): try: - return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True) + return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True).decode('utf-8') except: return "" # server seems dead @@ -189,7 +189,7 @@ def get_processlist(client_cmd): def get_stacktraces_from_gdb(server_pid): cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) try: - return subprocess.check_output(cmd, shell=True) + return subprocess.check_output(cmd, shell=True).decode('utf-8') except Exception as ex: return "Error occured while receiving stack traces from gdb: {}".format(str(ex)) @@ -198,7 +198,10 @@ def get_stacktraces_from_gdb(server_pid): # it does not work in Sandbox def get_stacktraces_from_clickhouse(client): try: - return subprocess.check_output("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True) + return subprocess.check_output("{} --allow_introspection_functions=1 --query " + "\"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), " + "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " + "FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8') except Exception as ex: return "Error occured while receiving stack traces from client: {}".format(str(ex)) diff --git a/tests/config/install.sh b/tests/config/install.sh index ef9604904e7..ff96e46c947 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -30,6 +30,7 @@ ln -sf $SRC_PATH/config.d/database_atomic.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/ +ln -sf $SRC_PATH/users.d/database_atomic_drop_detach_sync.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/ints_dictionary.xml $DEST_SERVER_PATH/ ln -sf $SRC_PATH/strings_dictionary.xml $DEST_SERVER_PATH/ diff --git a/tests/config/users.d/database_atomic_drop_detach_sync.xml b/tests/config/users.d/database_atomic_drop_detach_sync.xml new file mode 100644 index 00000000000..4313edf8be1 --- /dev/null +++ b/tests/config/users.d/database_atomic_drop_detach_sync.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/README.md b/tests/integration/README.md index bc64b686782..0886dc2cfac 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -14,7 +14,32 @@ Don't use Docker from your system repository. * [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev` * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest` -* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: `sudo -H pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio confluent-kafka avro +* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: + +``` +sudo -H pip install \ + PyMySQL \ + aerospike \ + avro \ + cassandra-driver \ + confluent-kafka \ + dicttoxml \ + docker \ + docker-compose==1.22.0 \ + grpcio \ + grpcio-tools \ + kafka-python \ + kazoo \ + minio \ + protobuf \ + psycopg2-binary==2.7.5 \ + pymongo \ + pytest \ + pytest-timeout \ + redis \ + tzlocal \ + urllib3 +``` (highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio` diff --git a/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml b/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/tests/integration/test_concurrent_ttl_merges/test.py b/tests/integration/test_concurrent_ttl_merges/test.py index f067e65f58a..65bc3828b38 100644 --- a/tests/integration/test_concurrent_ttl_merges/test.py +++ b/tests/integration/test_concurrent_ttl_merges/test.py @@ -5,8 +5,8 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml', 'configs/log_conf.xml'], with_zookeeper=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_disabled_mysql_server/__init__.py b/tests/integration/test_disabled_mysql_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_disabled_mysql_server/configs/remote_servers.xml b/tests/integration/test_disabled_mysql_server/configs/remote_servers.xml new file mode 100644 index 00000000000..de8e5865f12 --- /dev/null +++ b/tests/integration/test_disabled_mysql_server/configs/remote_servers.xml @@ -0,0 +1,12 @@ + + + + + + node1 + 9000 + + + + + diff --git a/tests/integration/test_disabled_mysql_server/test.py b/tests/integration/test_disabled_mysql_server/test.py new file mode 100644 index 00000000000..a2cbcb17534 --- /dev/null +++ b/tests/integration/test_disabled_mysql_server/test.py @@ -0,0 +1,60 @@ +import time +import contextlib +import pymysql.cursors +import pytest +import os +import subprocess + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster, get_docker_compose_path +from helpers.network import PartitionManager + +cluster = ClickHouseCluster(__file__) +clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +class MySQLNodeInstance: + def __init__(self, user='root', password='clickhouse', hostname='127.0.0.1', port=3308): + self.user = user + self.port = port + self.hostname = hostname + self.password = password + self.mysql_connection = None # lazy init + + def alloc_connection(self): + if self.mysql_connection is None: + self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, + port=self.port, autocommit=True) + return self.mysql_connection + + def query(self, execution_query): + with self.alloc_connection().cursor() as cursor: + cursor.execute(execution_query) + + def close(self): + if self.mysql_connection is not None: + self.mysql_connection.close() + + +def test_disabled_mysql_server(started_cluster): + with contextlib.closing(MySQLNodeInstance()) as mysql_node: + mysql_node.query("CREATE DATABASE test_db;") + mysql_node.query("CREATE TABLE test_db.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + + with PartitionManager() as pm: + clickhouse_node.query("CREATE DATABASE test_db ENGINE = MySQL('mysql1:3306', 'test_db', 'root', 'clickhouse')") + + pm._add_rule({'source': clickhouse_node.ip_address, 'destination_port': 3306, 'action': 'DROP'}) + clickhouse_node.query("SELECT * FROM system.parts") + clickhouse_node.query("SELECT * FROM system.mutations") + clickhouse_node.query("SELECT * FROM system.graphite_retentions") + + clickhouse_node.query("DROP DATABASE test_db") diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index c748653bc82..ad09519a484 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -21,8 +21,8 @@ def cluster(): def test_different_types(cluster): node = cluster.instances["node"] - responce = node.query("SELECT * FROM system.disks") - disks = responce.split("\n") + response = node.query("SELECT * FROM system.disks") + disks = response.split("\n") for disk in disks: if disk == '': # skip empty line (after split at last position) continue diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py index e7b86a210bd..0758dc38ba7 100644 --- a/tests/integration/test_distributed_load_balancing/test.py +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -26,7 +26,7 @@ def bootstrap(): # just after server starts (+ 2 seconds, reload timeout). # # And on configuration reload the clusters will be re-created, so some - # internal stuff will be reseted: + # internal stuff will be reset: # - error_count # - last_used (round_robing) # diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 496b34f22f0..07478d99657 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -286,7 +286,7 @@ def test_query_parser(start_cluster): node1.query( "ALTER TABLE table_with_normal_policy MODIFY SETTING storage_policy='moving_jbod_with_external'") finally: - node1.query("DROP TABLE IF EXISTS table_with_normal_policy") + node1.query("DROP TABLE IF EXISTS table_with_normal_policy SYNC") @pytest.mark.parametrize("name,engine", [ @@ -327,7 +327,7 @@ def test_alter_policy(start_cluster, name, engine): name=name)) == "jbods_with_external\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def get_random_string(length): @@ -355,9 +355,7 @@ def test_no_warning_about_zero_max_data_part_size(start_cluster): ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' """) - node.query(""" - DROP TABLE default.test_warning_table - """) + node.query("DROP TABLE default.test_warning_table SYNC") log = get_log(node) assert not re.search("Warning.*Volume.*special_warning_zero_volume", log) assert not re.search("Warning.*Volume.*special_warning_default_volume", log) @@ -398,7 +396,7 @@ def test_round_robin(start_cluster, name, engine): assert used_disks[0] != used_disks[1] assert used_disks[2] == used_disks[0] finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -425,7 +423,7 @@ def test_max_data_part_size(start_cluster, name, engine): assert len(used_disks) == 1 assert used_disks[0] == 'external' finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -451,8 +449,11 @@ def test_jbod_overflow(start_cluster, name, engine): data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + for p in ("/jbod1", "/jbod2", "/external"): + print(node1.exec_in_container([f"bash", "-c", f"find {p} | xargs -n1 du -sh"])) + used_disks = get_used_disks_for_table(node1, name) - assert all(disk == 'jbod1' for disk in used_disks) + assert set(used_disks) == {'jbod1'} # should go to the external disk (jbod is overflown) data = [] # 10MB in total @@ -461,6 +462,9 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + for p in ("/jbod1", "/jbod2", "/external"): + print(node1.exec_in_container([f"bash", "-c", f"find {p} | xargs -n1 du -sh"])) + used_disks = get_used_disks_for_table(node1, name) assert used_disks[-1] == 'external' @@ -478,7 +482,7 @@ def test_jbod_overflow(start_cluster, name, engine): assert all(disk == 'external' for disk in disks_for_merges) finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -495,6 +499,8 @@ def test_background_move(start_cluster, name, engine): SETTINGS storage_policy='moving_jbod_with_external' """.format(name=name, engine=engine)) + node1.query(f"SYSTEM START MERGES {name}") + for i in range(5): data = [] # 5MB in total for i in range(5): @@ -523,8 +529,10 @@ def test_background_move(start_cluster, name, engine): # first (oldest) part was moved to external assert path.startswith("/external") + node1.query(f"SYSTEM START MERGES {name}") + finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -611,7 +619,7 @@ def test_start_stop_moves(start_cluster, name, engine): assert used_disks[0] == 'external' finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def get_path_for_part_from_part_log(node, table, part_name): @@ -699,7 +707,7 @@ def test_alter_move(start_cluster, name, engine): assert node1.query("SELECT COUNT() FROM {}".format(name)) == "4\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("volume_or_disk", [ @@ -748,7 +756,7 @@ def test_alter_move_half_of_partition(start_cluster, volume_or_disk): assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("volume_or_disk", [ @@ -792,7 +800,7 @@ def test_alter_double_move_partition(start_cluster, volume_or_disk): volume_or_disk=volume_or_disk)) finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def produce_alter_move(node, name): @@ -876,7 +884,7 @@ def test_concurrent_alter_move(start_cluster, name, engine): assert node1.query("SELECT 1") == "1\n" assert node1.query("SELECT COUNT() FROM {}".format(name)) == "500\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -929,7 +937,7 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): assert node1.query("SELECT 1") == "1\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -960,7 +968,7 @@ def test_detach_attach(start_cluster, name, engine): assert node1.query("SELECT count() FROM {}".format(name)).strip() == "5" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -1006,7 +1014,7 @@ def test_mutate_to_another_disk(start_cluster, name, engine): finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -1064,7 +1072,7 @@ def test_concurrent_alter_modify(start_cluster, name, engine): assert node1.query("SELECT COUNT() FROM {}".format(name)) == "100\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def test_simple_replication_and_moves(start_cluster): @@ -1131,7 +1139,7 @@ def test_simple_replication_and_moves(start_cluster): set(disks2) == set(["jbod1", "external"]) finally: for node in [node1, node2]: - node.query("DROP TABLE IF EXISTS replicated_table_for_moves") + node.query("DROP TABLE IF EXISTS replicated_table_for_moves SYNC") def test_download_appropriate_disk(start_cluster): @@ -1165,7 +1173,7 @@ def test_download_appropriate_disk(start_cluster): finally: for node in [node1, node2]: - node.query("DROP TABLE IF EXISTS replicated_table_for_download") + node.query("DROP TABLE IF EXISTS replicated_table_for_download SYNC") def test_rename(start_cluster): @@ -1202,9 +1210,9 @@ def test_rename(start_cluster): node1.query("SELECT COUNT() FROM default.renaming_table1") finally: - node1.query("DROP TABLE IF EXISTS default.renaming_table") - node1.query("DROP TABLE IF EXISTS default.renaming_table1") - node1.query("DROP TABLE IF EXISTS test.renaming_table2") + node1.query("DROP TABLE IF EXISTS default.renaming_table SYNC") + node1.query("DROP TABLE IF EXISTS default.renaming_table1 SYNC") + node1.query("DROP TABLE IF EXISTS test.renaming_table2 SYNC") def test_freeze(start_cluster): @@ -1238,7 +1246,7 @@ def test_freeze(start_cluster): node1.exec_in_container(["bash", "-c", "find /external/shadow -name '*.mrk2' | grep '.*'"]) finally: - node1.query("DROP TABLE IF EXISTS default.freezing_table") + node1.query("DROP TABLE IF EXISTS default.freezing_table SYNC") node1.exec_in_container(["rm", "-rf", "/jbod1/shadow", "/external/shadow"]) @@ -1282,7 +1290,7 @@ def test_kill_while_insert(start_cluster): finally: try: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") except: """ClickHouse may be inactive at this moment and we don't want to mask a meaningful exception.""" @@ -1343,7 +1351,7 @@ def test_move_while_merge(start_cluster): assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["2"] finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def test_move_across_policies_does_not_work(start_cluster): @@ -1384,8 +1392,8 @@ def test_move_across_policies_does_not_work(start_cluster): assert node1.query("""SELECT * FROM {name}""".format(name=name)).splitlines() == ["1"] finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) - node1.query("DROP TABLE IF EXISTS {name}2".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") + node1.query(f"DROP TABLE IF EXISTS {name}2 SYNC") def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, parts_after_cmds): @@ -1420,7 +1428,7 @@ def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, pa assert len(parts) == parts_after_cmds finally: - node.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node.query(f"DROP TABLE IF EXISTS {name} SYNC") def _check_merges_are_working(node, storage_policy, volume, shall_work): @@ -1458,7 +1466,7 @@ def _check_merges_are_working(node, storage_policy, volume, shall_work): assert len(parts) == 1 if shall_work else created_parts finally: - node.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node.query(f"DROP TABLE IF EXISTS {name} SYNC") def _get_prefer_not_to_merge_for_storage_policy(node, storage_policy): diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index ab44d0ebea0..d7f98d5cb77 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -537,14 +537,14 @@ def test_rabbitmq_big_message(rabbitmq_cluster): @pytest.mark.timeout(420) def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): NUM_CONSUMERS = 10 - NUM_QUEUES = 2 + NUM_QUEUES = 10 instance.query(''' CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'test_sharding', - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_num_consumers = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -617,7 +617,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): rabbitmq_exchange_name = 'combo', rabbitmq_queue_base = 'combo', rabbitmq_num_consumers = 2, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 5, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; ''') @@ -879,7 +879,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_queue_base = 'over', rabbitmq_exchange_type = 'direct', rabbitmq_num_consumers = 5, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_max_block_size = 10000, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', @@ -1722,7 +1722,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'consumer_reconnect', rabbitmq_num_consumers = 10, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; ''') diff --git a/tests/integration/test_system_flush_logs/__init__.py b/tests/integration/test_system_flush_logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py b/tests/integration/test_system_flush_logs/test.py similarity index 100% rename from tests/integration/test_SYSTEM_FLUSH_LOGS/test.py rename to tests/integration/test_system_flush_logs/test.py diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py index 7f5bce97805..b159e8b4cf3 100644 --- a/tests/integration/test_system_queries/test.py +++ b/tests/integration/test_system_queries/test.py @@ -107,7 +107,7 @@ def test_RELOAD_CONFIG_AND_MACROS(started_cluster): assert TSV(instance.query("select * from system.macros")) == TSV("instance\tch1\nmac\tro\n") -def test_SYSTEM_FLUSH_LOGS(started_cluster): +def test_system_flush_logs(started_cluster): instance = cluster.instances['ch1'] instance.query(''' SET log_queries = 0; diff --git a/tests/queries/0_stateless/00273_quantiles.reference b/tests/queries/0_stateless/00273_quantiles.reference index d75d11191f4..616e06841e4 100644 --- a/tests/queries/0_stateless/00273_quantiles.reference +++ b/tests/queries/0_stateless/00273_quantiles.reference @@ -6,17 +6,17 @@ [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] [0,0.50100005,9.51,49.55,99.6,199.7,299.8,399.9,500,600.1,700.2,800.3,900.4,950.45,990.49,999.499,1000] [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] -1 333334 [699140.3,835642,967430.8] [699999,833333,966666] +1 333334 [699144.2,835663,967429.2] [699999,833333,966666] 2 266667 [426549.5,536255.5,638957.6] [426665,533332,639999] -3 114285 [296938.5,342335,388777.5] [297142,342856,388570] +3 114285 [296938,342324,388778] [297142,342856,388570] 4 63492 [228370.2,254019.5,279351.4] [228571,253968,279364] 5 40404 [185603.4,202009,218107] [185858,202020,218181] -6 27972 [156598.6,167864,179118.40000000002] [156643,167832,179020] -7 20513 [135401,143553.5,151792.5] [135384,143589,151794] +6 27972 [156598.7,167866,179118.3] [156643,167832,179020] +7 20513 [135400.8,143550,151792.6] [135384,143589,151794] 8 15686 [119239.20000000001,125463,131772.40000000002] [119215,125490,131764] -9 12384 [106509.79999999999,111538,116415.8] [106501,111455,116408] -10 10025 [96223.6,100347,104288.6] [96240,100250,104260] -11 8282 [87732.70000000001,91035,94408.6] [87784,91097,94409] +9 12384 [106510.20000000001,111539,116415.7] [106501,111455,116408] +10 10025 [96223.2,100346,104288.7] [96240,100250,104260] +11 8282 [87732.8,91036,94410.20000000001] [87784,91097,94409] 12 6957 [80694.6,83477,86259.4] [80694,83477,86260] 13 5925 [74666.40000000001,77036,79405.6] [74666,77036,79406] 14 5109 [69475.8,71519,73562.2] [69475,71519,73563] diff --git a/tests/queries/0_stateless/00932_geohash_support.sql b/tests/queries/0_stateless/00932_geohash_support.sql index a477332b532..aeed72176b9 100644 --- a/tests/queries/0_stateless/00932_geohash_support.sql +++ b/tests/queries/0_stateless/00932_geohash_support.sql @@ -45,7 +45,7 @@ select 12 as p, geohashEncode(longitude, latitude, p) as actual, if(actual = enc -- Here results are floats, and hence may not be compared for equality directly. -- We select all values that are off by some reasonable value: --- each byte of encoded string provides 5 bits of precison, (roughly 2.5 for lon and lat) +-- each byte of encoded string provides 5 bits of precision, (roughly 2.5 for lon and lat) -- each bit of precision divides value range by 2. -- hence max error is roughly value range 2.5 times divided by 2 for each precision bit. -- initial value range is [-90..90] for latitude and [-180..180] for longitude. diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 9f4bfb2c436..173bf44e1f1 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -10,18 +10,18 @@ $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01107.mt (n UInt64) ENGINE=MergeTree() $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(3) FROM numbers(5)" & sleep 1 -$CLICKHOUSE_CLIENT -q "DETACH TABLE test_01107.mt" +$CLICKHOUSE_CLIENT -q "DETACH TABLE test_01107.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" 2>&1 | grep -F "Code: 57" > /dev/null && echo "OK" -$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK" +$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK" wait $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" -$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" +$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "ATTACH DATABASE test_01107" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" & sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" && sleep 1 && echo "dropped" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 && sleep 1 && echo "dropped" wait diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index d23be44e784..d220491d152 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -36,9 +36,10 @@ $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_3.mt_tmp" $CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_3" -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01114_2.mt UUID '00001114-0000-4000-8000-000000000002' (n UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY (n % 5)" -$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" -$CLICKHOUSE_CLIENT -q "SELECT name, uuid, create_table_query FROM system.tables WHERE database='test_01114_2'" +explicit_uuid=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01114_2.mt UUID '$explicit_uuid' (n UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY (n % 5)" +$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" +$CLICKHOUSE_CLIENT -q "SELECT name, uuid, create_table_query FROM system.tables WHERE database='test_01114_2'" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" $CLICKHOUSE_CLIENT -q "SELECT count(col), sum(col) FROM (SELECT n + sleepEachRow(1.5) AS col FROM test_01114_1.mt)" & # 30s, result: 20, 190 @@ -55,18 +56,18 @@ $CLICKHOUSE_CLIENT -q "EXCHANGE TABLES test_01114_1.mt AND test_01114_2.mt" $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" uuid_mt1=$($CLICKHOUSE_CLIENT -q "SELECT uuid FROM system.tables WHERE database='test_01114_1' AND name='mt'") $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_1.mt" | sed "s/$uuid_mt1/00001114-0000-4000-8000-000000000001/g" -$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" +$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" -$CLICKHOUSE_CLIENT -q "DROP TABLE test_01114_1.mt" +$CLICKHOUSE_CLIENT -q "DROP TABLE test_01114_1.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01114_1.mt (s String) ENGINE=Log()" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01114_1.mt SELECT 's' || toString(number) FROM numbers(5)" $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" # result: 5 $CLICKHOUSE_CLIENT -q "SELECT tuple(s, sleepEachRow(3)) FROM test_01114_1.mt" > /dev/null & # 15s sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" && echo "dropped" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" --database_atomic_wait_for_drop_and_detach_synchronously=0 && echo "dropped" wait # for INSERT $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01114_2.mt" # result: 30, 435 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_2" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_2" --database_atomic_wait_for_drop_and_detach_synchronously=0 diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index 0ee583a7265..319b537e84b 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -49,4 +49,4 @@ $CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(query_duration_ms) < $max_time_ms $CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" -$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" +$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" --database_atomic_wait_for_drop_and_detach_synchronously=0 diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh index b14defd672a..5333d0b4b0b 100755 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Regression for MemoryTracker that had been incorrectly accounted -# (it was reseted before deallocation) +# (it was reset before deallocation) # # For this will be used: # - two-level group by diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql index babc8457bc0..b8b86384bc4 100644 --- a/tests/queries/0_stateless/01322_student_ttest.sql +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS student_ttest; CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (0.88854,-2.90702), (-5.76966,3.61651), (6.76618,4.27458), (3.55546,4.82133), (-9.76948,9.59483), (4.92323,1.00424), (-0.36352,2.04147), (0.97018,-3.58214), (4.61656,6.59543), (-6.78292,-1.00532), (4.02008,-3.59794), (12.41838,-2.82434), (5.14417,-3.13194), (3.86836,9.90977), (-1.26199,0.523), (12.44106,4.62779), (3.28349,-2.56872), (1.77261,2.25807), (-8.94748,1.04044), (-1.01449,-2.35744), (-1.26377,10.81531), (6.79682,-9.68469), (6.32333,3.80885), (-8.21214,12.70435), (-1.68565,-6.01112), (9.7557,1.89065), (3.66694,5.08892), (1.39967,3.45254), (-5.52035,11.58151), (-10.95601,0.85035), (0.93877,8.38397), (1.45933,1.17169), (-5.40551,4.74621), (-0.83857,-1.66614), (8.50794,4.2414), (-6.68686,1.68765), (5.03099,1.85223), (1.56251,9.10111), (4.17381,-2.38085), (-2.92644,-14.79595), (5.11068,-3.8938), (2.09617,-3.41864), (11.7787,-3.15282), (6.50336,-0.56684), (0.62098,12.87997), (-7.97121,6.89115), (3.81902,12.921), (0.33151,-7.94908), (10.68584,2.45687), (0.56007,2.14957), (-7.38621,7.55081), (5.05882,-3.71534), (2.34616,-2.41064), (11.3806,-0.80734), (5.95276,-4.75651), (-3.01429,2.05241), (5.98169,-5.44523), (0.96985,-2.75054), (-1.15932,-13.00131), (2.11547,-2.74451), (2.49668,-1.39004), (-12.49569,-3.02854), (-4.94667,7.65112), (-3.64215,1.1245), (-8.35595,6.74117), (3.211,-0.75777), (2.33805,8.93451), (2.38608,-8.85559), (-3.2862,-0.36405), (-0.80454,4.02742), (-0.53483,6.88718), (10.66445,-1.05124), (-0.37619,3.04085), (0.48246,3.32368), (7.41919,1.147), (0.42414,3.41554), (-2.32335,-3.47851), (-0.70223,-0.47684), (-5.9332,-0.55605), (-1.20561,-0.17006), (3.39865,2.26218), (9.61739,12.45494), (-0.78651,-1.84097), (-4.00256,1.64934), (-7.99646,-7.07496), (8.72923,-9.99462), (0.71859,6.09954), (-1.62726,-1.05319), (5.11234,3.04757), (-0.95625,0.93899), (-3.75573,-4.63243), (1.03141,-7.43322), (-3.33588,-7.298), (1.51804,-6.59016), (-3.30935,-6.11649), (-1.97507,0.56682), (4.06456,2.00661), (3.27195,-2.79814), (-7.81761,2.84482), (-3.81785,3.65348), (-4.18311,-4.22807), (-11.33313,-4.54336), (-0.25221,-3.63343), (7.2514,2.96878), (5.30301,6.11661), (2.46762,-1.70919), (4.22716,-4.71133), (0.33916,6.09652), (9.7638,-6.83454), (-7.58684,0.18006), (-4.09888,1.51676), (4.26617,-5.31646), (-0.56744,-3.21215), (4.65125,-5.07599), (-1.30301,-2.36591), (4.53771,3.55724), (9.96929,4.8904), (3.72939,-3.22586), (-2.29818,-1.74928), (3.09417,5.73458), (0.82251,1.41188), (5.29975,2.86255), (2.8685,2.90179), (-5.73321,-2.19949), (-1.85651,1.72727), (-1.07984,1.76939), (9.78342,-0.12848), (-13.49652,-0.52), (3.68791,3.48333), (1.9998,7.8262), (1.11674,0.09099), (9.43869,7.77017), (4.07029,9.49484), (5.32715,1.42825), (7.16504,1.99624), (6.66096,4.00419), (-5.7111,1.07925), (-0.38575,-0.09987), (4.49165,-5.48733), (-3.36489,-1.83517), (7.71814,2.38059), (-1.58966,1.42075), (-1.61063,-1.11968), (-0.91602,-6.46035), (0.73459,7.66576), (-3.24463,4.6307), (6.3947,5.55989), (-2.77845,3.16684), (4.45899,5.07671), (-8.84186,-10.20566), (2.62276,-4.73386), (1.774,1.28353), (4.3692,6.75679), (0.05942,12.09895), (-1.44042,7.0049), (-2.53594,7.16156), (-2.24752,-0.64311), (4.98874,-0.66747), (4.05434,3.99996), (-2.56483,9.07298), (-6.79286,-4.60971), (-2.06165,0.70744), (-0.26056,2.56774), (1.89567,9.32424), (-3.15145,3.95087), (-7.31321,7.11372), (0.28936,-0.89284), (-0.63111,8.6155), (0.22611,-0.14141), (-9.3377,-4.86319), (-5.76638,-6.95801), (3.87306,4.44883), (6.7011,4.6156), (9.03915,-2.3579), (-1.21835,-5.1186), (0.82892,8.12819), (2.80656,2.78392), (-1.34746,-4.30221), (-1.99912,-1.47506), (0.6036,6.8598), (-3.46117,0.47636), (5.23732,0.95383), (-1.86702,7.79779), (-5.86115,-2.61767), (6.48523,-10.5087), (-7.40158,-2.74299), (-1.38913,3.87369), (4.94613,-1.07093), (-2.07818,4.98864), (2.39808,-7.50772), (4.89238,6.41316), (4.39481,1.39061), (5.20425,-3.1747), (13.62598,-2.13621), (-2.86293,-0.02203), (-3.62396,0.89025), (-4.28695,-5.87746), (4.66425,3.60026), (2.20871,-0.23178), (1.60382,-2.1897), (-9.87024,-5.85101), (-7.37302,-1.6053), (-4.17814,3.6184), (2.5148,-8.53795), (3.21708,-0.35987), (-11.48089,2.15301), (1.19821,-6.60692), (-0.07436,9.54341), (-1.10652,1.11511), (4.03395,2.94025), (-4.35883,12.05657), (2.04013,3.75156), (0.52264,7.95597), (8.14004,-0.99449), (-8.86949,0.90597), (-0.35807,-7.90627), (-10.71113,3.50863), (-2.13755,-1.47493), (0.50715,4.11671), (6.30826,10.06325), (2.37527,-1.06059), (0.20872,-1.37737), (-5.85729,-0.42542), (-4.97217,-3.90267), (-9.78434,9.35037), (-1.53277,-7.91219), (0.14827,-4.69945), (-1.053,3.63776), (1.74558,3.46492), (11.17194,2.84518), (9.35487,-3.04301), (-9.17209,8.82764), (10.41814,7.80134), (7.41206,7.87755), (3.71775,7.01035), (-2.04674,2.43271), (6.18037,11.36418), (5.6383,-6.92659), (-0.90058,5.95541), (-1.27073,3.59436), (-2.3473,5.18429), (-8.44271,4.20225), (2.75551,0.5029), (-1.15521,4.03074), (4.08722,5.23152), (-1.70399,10.65409), (7.24114,-0.69845), (-8.43976,11.70096), (-1.53052,5.80692), (-0.00526,-8.1819), (-4.04813,4.31485), (-2.84299,5.7227), (-5.201,5.67398), (7.75774,-1.75826), (-2.85791,7.54164), (-3.86071,-1.79026), (-1.80029,-1.7395), (-5.26015,5.65042), (-3.158,0.38765), (7.71014,-4.64719), (-4.84866,-10.22048), (-8.38785,-2.05447), (7.67021,-2.43441), (4.96521,-5.38551), (-0.40919,5.47764), (-3.25711,8.26637), (3.07685,-3.6421), (2.89376,-11.66269), (-10.47331,3.972), (-3.48942,5.46642), (1.13906,-3.72304), (-8.57454,5.75251), (-3.38963,5.12841), (-2.3195,0.59067), (-1.60694,5.21138), (-5.57406,-4.58702), (-0.93075,-8.737), (-11.76579,-2.12737), (10.68283,0.22888), (8.74324,-1.46448), (7.66409,2.40311), (4.76715,-5.21814), (0.44539,13.94749), (-1.35941,-2.77448), (4.18849,-3.7867), (-6.17097,3.4954), (0.27977,3.12586), (-1.45006,-7.01485), (-4.81694,-3.20727), (-3.0297,6.31415), (0.02145,2.37521), (2.46883,8.13787), (9.60317,2.15956), (-9.93898,-0.40842), (1.05549,-7.27283), (5.55366,4.27575), (-3.80722,-2.89126), (-4.18851,6.84344), (1.00351,7.0869), (3.11385,-5.18837), (-5.17623,2.67648), (-3.18396,-6.57021), (-6.65302,0.60429), (-0.50832,-1.04921), (-4.04375,7.12873), (4.52707,1.68973), (6.63124,-2.58404), (-3.72082,-3.83114), (5.79825,-7.26546), (-2.0158,-5.07153), (-2.78369,-0.80395), (-1.91821,2.09455), (6.31714,4.33374), (-1.80869,8.54335), (8.55586,0.80566), (2.40826,-8.38085), (-8.46361,7.54812), (5.04452,8.78007), (-0.84665,1.5857), (2.30903,8.43855), (-3.71837,-1.90846), (-0.69419,-1.2434), (3.6733,7.16172), (-1.96098,-3.44129), (2.36747,-6.37542), (-12.03622,-4.99486), (4.38481,4.99033), (2.93955,-1.83734), (2.16804,-2.83289), (-0.08218,-4.13997), (-3.97934,1.40163), (-7.43985,8.57867), (0.91666,-1.87639), (7.23432,3.41667), (-6.13303,6.31762), (-10.23217,1.58473), (-6.21681,1.63625), (-0.80934,-6.93618), (0.17914,3.58046), (2.13338,-6.8097), (6.97656,4.69978), (6.90455,-1.72912), (6.25943,5.29491), (-6.04019,-1.63062), (-7.30909,5.83818), (1.4589,17.0769), (12.00208,4.54301), (2.22457,-1.33801), (-2.45912,5.64339), (-6.92213,1.26913), (4.05547,-1.01553), (0.04709,4.8316), (-7.70952,3.08635), (-1.47883,-2.27738), (1.3701,-1.13761), (-4.92928,10.08698), (-2.75872,5.33827), (-0.09178,2.84345), (2.62642,-1.51132), (-1.14623,13.46078), (2.76609,8.58965), (4.94404,-2.36683), (-7.01764,-1.8217), (-10.91568,1.96981), (-2.49738,2.31718), (0.73576,3.66493), (2.25436,1.93104), (-1.72956,5.20332), (2.41054,3.20519), (5.72149,3.34631), (-6.41371,7.0087), (3.38217,-7.96126), (1.24133,-0.62182), (10.03634,-4.65227), (-2.37303,10.6572), (-1.35543,4.50891), (-1.4387,9.74298), (-4.0976,3.85707), (-0.82501,6.41144), (-1.93498,1.48649), (5.59955,2.28076), (5.46656,2.75342), (2.43568,-5.40401), (-0.23926,7.11389), (-4.9945,5.74368), (-4.96655,6.78345), (-0.59258,3.83773), (2.02497,0.70959), (0.67583,0.57434), (3.16522,1.5888), (-1.9673,3.94889), (-6.75319,5.8234), (-6.69723,7.78366), (0.81148,9.08354), (4.44531,-7.99182), (-4.43522,-2.77033), (-5.28602,-10.29342), (-3.58829,1.76251), (-7.97395,2.09266), (-2.84891,4.20614), (-3.95112,-3.63064), (3.54945,-2.17794), (12.12376,-2.66225), (-3.12347,-2.74707), (3.65209,-1.93431), (9.34031,1.38629), (-0.26348,4.12816), (-5.23968,-1.58902), (2.22336,-5.08864), (-10.70405,-2.30491), (-4.41319,2.64605), (-5.94912,1.16158), (1.8147,2.63534), (7.69287,1.4956), (9.46125,-4.60768), (4.72497,0.60771), (-0.57565,3.29549), (-1.12303,-1.42592), (2.90272,0.8883), (-4.4584,-1.10612), (4.28819,-2.57296), (11.64512,5.88085), (-1.80395,7.40745), (2.51605,13.48116), (-3.18439,5.53539), (-0.70213,-1.46014), (-7.68383,3.73304), (-8.32268,3.5435), (-8.71115,-3.89151), (9.96933,4.16265), (0.95675,2.32663), (3.35114,5.31735), (-2.66008,6.33485), (7.75456,2.1339), (0.73568,0.82708), (0.3483,-2.95155), (-1.09203,-6.76019), (-7.76963,-4.20179), (5.81902,8.78354), (-3.41424,1.41863), (-0.39209,7.65689), (4.67608,-6.52601), (0.68753,-4.4426), (5.17179,-4.49483), (4.98983,-3.91479), (-0.12659,-2.84562), (3.25267,2.58974), (1.50184,2.24424), (2.94507,-4.65846), (-0.42333,8.4062), (-3.66227,8.20262), (8.90812,-8.63752), (4.74411,4.97966), (2.22018,-0.35563), (-2.07976,-4.72116), (4.8711,-2.95997), (0.5023,2.73959), (6.31569,-0.23956), (-4.36903,10.13915), (3.82146,11.83775), (-6.99477,-2.50332), (3.61225,-0.58181), (14.69335,-7.62836), (0.58368,2.26478), (4.65341,-3.50179), (-3.14272,-2.08023), (2.67048,4.07256), (4.64963,-1.40826), (-2.70828,-2.33644), (1.42923,3.00197), (5.84498,4.23668), (-4.76568,-2.24647), (0.19907,1.0445), (1.67486,-0.31901), (5.32145,8.62657), (-8.03477,3.92817), (3.46776,0.08462), (4.66374,10.15884), (-5.37394,0.4113), (5.39045,4.45847), (-1.44756,5.82941), (-1.64419,6.59202), (3.39699,-3.73441), (-2.94659,-5.86969), (-2.38437,-4.56543), (-0.23958,-1.32636), (6.88389,-0.17884), (-2.7172,-3.56181), (-1.53419,-0.66932), (7.38841,6.87538), (-5.44178,0.73527), (-0.89287,-0.24177), (2.93546,-0.8657), (-0.26901,-0.22977), (-4.70044,1.02095), (2.25846,6.16311), (-9.28813,-5.68027), (6.04268,-3.7619), (4.41693,4.22959), (1.75714,-1.5249); SELECT '-2.610898982580138', '0.00916587538237954'; -SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -15,5 +15,5 @@ DROP TABLE IF EXISTS student_ttest; CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); SELECT '-28.740781574102936', '7.667329672103986e-133'; -SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 5a5b52ab612..cce65c28bd8 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -3,19 +3,19 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (27.5,27.1), (21.0,22.0), (19.0,20.8), (23.6,23.4), (17.0,23.4), (17.9,23.5), (16.9,25.8), (20.1,22.0), (21.9,24.8), (22.6,20.2), (23.1,21.9), (19.6,22.1), (19.0,22.9), (21.7,20.5), (21.4,24.4); SELECT '0.021378001462867'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (30.02,29.89), (29.99,29.93), (30.11,29.72), (29.97,29.98), (30.01,30.02), (29.99,29.98); SELECT '0.090773324285671'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (0.010268,0.159258), (0.000167,0.136278), (0.000167,0.122389); SELECT '0.00339907162713746'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -24,7 +24,7 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); SELECT '-0.5028215369186904', '0.6152361677168877'; -SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -33,5 +33,5 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); SELECT '14.971190998235835', '5.898143508382202e-44'; -SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference new file mode 100644 index 00000000000..420bb17ae8b --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -0,0 +1,10 @@ +===http=== +{"total spans":"4","unique spans":"4","unique non-zero parent spans":"2"} +{"initial query spans with proper parent":"1"} +{"unique non-empty tracestate values":"1"} +===native=== +{"total spans":"2","unique spans":"2","unique non-zero parent spans":"2"} +{"initial query spans with proper parent":"1"} +{"unique non-empty tracestate values":"1"} +===sampled=== +OK diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh new file mode 100755 index 00000000000..fe52b2b2032 --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -ue + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +function check_log +{ +${CLICKHOUSE_CLIENT} --format=JSONEachRow -nq " +system flush logs; + +-- Check the number of query spans with given trace id, to verify it was +-- propagated. +select count(*) "'"'"total spans"'"'", + uniqExact(span_id) "'"'"unique spans"'"'", + uniqExactIf(parent_span_id, parent_span_id != 0) + "'"'"unique non-zero parent spans"'"'" + from system.opentelemetry_span_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + ; + +-- Also check that the initial query span in ClickHouse has proper parent span. +select count(*) "'"'"initial query spans with proper parent"'"'" + from + (select *, attribute_name, attribute_value + from system.opentelemetry_span_log + array join attribute.names as attribute_name, + attribute.values as attribute_value) o + join system.query_log on query_id = o.attribute_value + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + and parent_span_id = reinterpretAsUInt64(unhex('73')) + and o.attribute_name = 'clickhouse.query_id' + and is_initial_query + and type = 'QueryFinish' + ; + +-- Check that the tracestate header was propagated. It must have exactly the +-- same non-empty value for all 'query' spans in this trace. +select uniqExact(value) "'"'"unique non-empty tracestate values"'"'" + from system.opentelemetry_span_log + array join attribute.names as name, attribute.values as value + where + trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + and name = 'clickhouse.tracestate' + and length(value) > 0 + ; +" +} + +# Generate some random trace id so that the prevous runs of the test do not interfere. +echo "===http===" +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +# Check that the HTTP traceparent is read, and then passed through `remote` +# table function. We expect 4 queries -- one initial, one SELECT and two +# DESC TABLE. Two DESC TABLE instead of one looks like a bug, see the issue: +# https://github.com/ClickHouse/ClickHouse/issues/14228 +${CLICKHOUSE_CURL} \ + --header "traceparent: 00-$trace_id-0000000000000073-01" \ + --header "tracestate: some custom state" "http://localhost:8123/" \ + --get \ + --data-urlencode "query=select 1 from remote('127.0.0.2', system, one) format Null" + +check_log + +# With another trace id, check that clickhouse-client accepts traceparent, and +# that it is passed through URL table function. We expect two query spans, one +# for the initial query, and one for the HTTP query. +echo "===native===" +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +${CLICKHOUSE_CLIENT} \ + --opentelemetry-traceparent "00-$trace_id-0000000000000073-01" \ + --opentelemetry-tracestate "another custom state" \ + --query "select * from url('http://127.0.0.2:8123/?query=select%201%20format%20Null', CSV, 'a int')" + +check_log + +# Test sampled tracing. The traces should be started with the specified +# probability, only for initial queries. +echo "===sampled===" +query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +for i in {1..200} +do + ${CLICKHOUSE_CLIENT} \ + --opentelemetry_start_trace_probability=0.1 \ + --query_id "$query_id-$i" \ + --query "select 1 from remote('127.0.0.2', system, one) format Null" \ + & + + # clickhouse-client is slow to start (initialization of DateLUT), so run + # several clients in parallel, but not too many. + if [[ $((i % 10)) -eq 0 ]] + then + wait + fi +done +wait + +${CLICKHOUSE_CLIENT} -q "system flush logs" +${CLICKHOUSE_CLIENT} -q " + with count(*) as c + -- expect 200 * 0.1 = 20 sampled events on average + select if(c > 5 and c < 35, 'OK', 'fail: ' || toString(c)) + from system.opentelemetry_span_log + array join attribute.names as name, attribute.values as value + where name = 'clickhouse.query_id' + and operation_name = 'query' + and parent_span_id = 0 -- only account for the initial queries + and value like '$query_id-%' + ; +" diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.expect b/tests/queries/0_stateless/01526_client_start_and_exit.expect new file mode 100755 index 00000000000..003439ffa54 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.expect @@ -0,0 +1,12 @@ +#!/usr/bin/expect -f + +log_user 1 +set timeout 5 +match_max 100000 + +if ![info exists env(CLICKHOUSE_PORT_TCP)] {set env(CLICKHOUSE_PORT_TCP) 9000} + +spawn bash -c "clickhouse-client --port $env(CLICKHOUSE_PORT_TCP) && echo $?" +expect ":) " +send -- "\4" +expect eof diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.reference b/tests/queries/0_stateless/01526_client_start_and_exit.reference new file mode 100644 index 00000000000..e3e2e7b22af --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.reference @@ -0,0 +1 @@ +Loaded 10000 queries. diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh new file mode 100755 index 00000000000..c179be79d03 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# Create a huge amount of tables, so Suggest will take a time to load +${CLICKHOUSE_CLIENT} -q "SELECT 'CREATE TABLE test_' || hex(randomPrintableASCII(40)) || '(x UInt8) Engine=Memory;' FROM numbers(10000)" --format=TSVRaw | ${CLICKHOUSE_BENCHMARK} -c32 -i 10000 -d 0 2>&1 | grep -F 'Loaded 10000 queries' + +function stress() +{ + while true; do + "${CURDIR}"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' + done +} + +export CURDIR +export -f stress + +for _ in {1..10}; do + timeout 3 bash -c stress & +done + +wait diff --git a/tests/queries/0_stateless/01526_initial_query_id.reference b/tests/queries/0_stateless/01526_initial_query_id.reference new file mode 100644 index 00000000000..e8d2c31aa17 --- /dev/null +++ b/tests/queries/0_stateless/01526_initial_query_id.reference @@ -0,0 +1,2 @@ +1 1 +2 1 diff --git a/tests/queries/0_stateless/01526_initial_query_id.sh b/tests/queries/0_stateless/01526_initial_query_id.sh new file mode 100755 index 00000000000..c5459625023 --- /dev/null +++ b/tests/queries/0_stateless/01526_initial_query_id.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ue + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +${CLICKHOUSE_CLIENT} -q "select 1 format Null" "--query_id=$query_id" + +${CLICKHOUSE_CURL} \ + --header "X-ClickHouse-Query-Id: $query_id" \ + "http://localhost:8123/" \ + --get \ + --data-urlencode "query=select 1 format Null" + +${CLICKHOUSE_CLIENT} -n -q " +system flush logs; +select interface, initial_query_id = query_id + from system.query_log + where query_id = '$query_id' and type = 'QueryFinish' + order by interface + ; +" + diff --git a/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference new file mode 100644 index 00000000000..041e329748e --- /dev/null +++ b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference @@ -0,0 +1,8 @@ +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +\N +00000000-0000-0000-0000-000000000000 +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +\N +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +00000000-0000-0000-0000-000000000000 diff --git a/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql new file mode 100644 index 00000000000..ae6a1b2db04 --- /dev/null +++ b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS to_uuid_test; + +SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'); +SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0T'); --{serverError 6} +SELECT toUUIDOrNull('61f0c404-5cb3-11e7-907b-a6006ad3dba0T'); +SELECT toUUIDOrZero('59f0c404-5cb3-11e7-907b-a6006ad3dba0T'); + +CREATE TABLE to_uuid_test (value String) ENGINE = TinyLog(); + +INSERT INTO to_uuid_test VALUES ('61f0c404-5cb3-11e7-907b-a6006ad3dba0'); +SELECT toUUID(value) FROM to_uuid_test; + +INSERT INTO to_uuid_test VALUES ('61f0c404-5cb3-11e7-907b-a6006ad3dba0T'); +SELECT toUUID(value) FROM to_uuid_test; -- {serverError 6} +SELECT toUUIDOrNull(value) FROM to_uuid_test; +SELECT toUUIDOrZero(value) FROM to_uuid_test; + +DROP TABLE to_uuid_test; + diff --git a/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.reference b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh new file mode 100755 index 00000000000..f341fbcdd9b --- /dev/null +++ b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# in case when clickhouse-local can't use temp folder it will try to create +# temporary subfolder in the current dir +TMP=/non-existent-folder-12123 ${CLICKHOUSE_LOCAL} -q 'SELECT 1' diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.reference b/tests/queries/0_stateless/01532_min_max_with_modifiers.reference new file mode 100644 index 00000000000..9e1683ca6f9 --- /dev/null +++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.reference @@ -0,0 +1,20 @@ +totals +1 1 1 +2 2 2 +3 3 3 + +0 1 3 +rollup +1 1 1 +2 2 2 +3 3 3 +0 1 3 +cube +1 1 1 +2 2 2 +3 3 3 +0 1 3 +======= +1 1 2 1 +2 2 3 1 +0 1 3 2 diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.sql b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql new file mode 100644 index 00000000000..0c8651c0f01 --- /dev/null +++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql @@ -0,0 +1,18 @@ +SELECT 'totals'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH TOTALS; +SELECT 'rollup'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH ROLLUP; +SELECT 'cube'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH CUBE; +SELECT '======='; + +SELECT + x, + min(x) AS lower, + max(x) + 1 AS upper, + upper - lower AS range +FROM +( + SELECT arrayJoin([1, 2]) AS x +) +GROUP BY x WITH ROLLUP; diff --git a/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql new file mode 100644 index 00000000000..4f32576e86b --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS bug_13492; + +CREATE TABLE bug_13492 (`d` DateTime) ENGINE = MergeTree +PARTITION BY toYYYYMMDD(d) ORDER BY tuple(); + +INSERT INTO bug_13492 SELECT addDays(now(), number) FROM numbers(100); + +SET max_threads = 5; + +SELECT DISTINCT 1 FROM bug_13492, numbers(1) n; + +SET max_threads = 2; + +SELECT DISTINCT 1 FROM bug_13492, numbers(1) n; + +DROP TABLE bug_13492; diff --git a/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference b/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference new file mode 100644 index 00000000000..e02acad09d6 --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference @@ -0,0 +1,4 @@ +442d3ff4-842a-45bb-8b02-b616122c0dc6 +05fe40cb-1d0c-45b0-8e60-8e311c2463f1 +2fc89389-4728-4b30-9e51-b5bc3ad215f6 +10000 diff --git a/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql b/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql new file mode 100644 index 00000000000..926739d3f58 --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS bug_14144; + +CREATE TABLE bug_14144 +( meta_source_req_uuid Nullable(UUID), + a Int64, + meta_source_type String +) +ENGINE = MergeTree +ORDER BY a; + +INSERT INTO bug_14144 SELECT cast(toUUID('442d3ff4-842a-45bb-8b02-b616122c0dc6'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +INSERT INTO bug_14144 SELECT cast(toUUIDOrZero('2fc89389-4728-4b30-9e51-b5bc3ad215f6'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +INSERT INTO bug_14144 SELECT cast(toUUIDOrNull('05fe40cb-1d0c-45b0-8e60-8e311c2463f1'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +SELECT DISTINCT meta_source_req_uuid +FROM bug_14144 +WHERE meta_source_type = 'missing' +ORDER BY meta_source_req_uuid ASC; + +TRUNCATE TABLE bug_14144; + +INSERT INTO bug_14144 SELECT generateUUIDv4(), number, 'missing' FROM numbers(10000); + +SELECT COUNT() FROM ( + SELECT DISTINCT meta_source_req_uuid + FROM bug_14144 + WHERE meta_source_type = 'missing' + ORDER BY meta_source_req_uuid ASC + LIMIT 100000 +); + +DROP TABLE bug_14144; + + + + diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference new file mode 100644 index 00000000000..231c72269ca --- /dev/null +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference @@ -0,0 +1 @@ +3998 diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql b/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql new file mode 100644 index 00000000000..c75e5dd501f --- /dev/null +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql @@ -0,0 +1 @@ +SELECT quantileDeterministic(number, sipHash64(number)) FROM remote('127.0.0.{1,2}', numbers(8193)); diff --git a/tests/queries/0_stateless/01534_lambda_array_join.reference b/tests/queries/0_stateless/01534_lambda_array_join.reference new file mode 100644 index 00000000000..78e098be306 --- /dev/null +++ b/tests/queries/0_stateless/01534_lambda_array_join.reference @@ -0,0 +1,3 @@ +[NULL] +[1,1] +\N 70 diff --git a/tests/queries/0_stateless/01534_lambda_array_join.sql b/tests/queries/0_stateless/01534_lambda_array_join.sql new file mode 100644 index 00000000000..aee9dd1411a --- /dev/null +++ b/tests/queries/0_stateless/01534_lambda_array_join.sql @@ -0,0 +1,12 @@ +SELECT arrayMap(x -> concat(x, concat(arrayJoin([1]), x, NULL), ''), [1]); +SELECT arrayMap(x -> arrayJoin([1]), [1, 2]); + +SELECT + arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomString(range(randomString(1048577), NULL), arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomString(range(NULL), 65537), 255)), range(1))), substring(randomString(NULL), x + 7), '257'), 1025)), range(7))) AS byte, + count() AS c + FROM numbers(10) + GROUP BY + arrayMap(x -> reinterpretAsUInt8(substring(randomString(randomString(range(randomString(255), NULL)), NULL))), range(3)), + randomString(range(randomString(1048577), NULL), NULL), + byte + ORDER BY byte ASC; diff --git a/tests/queries/0_stateless/01536_fuzz_cast.reference b/tests/queries/0_stateless/01536_fuzz_cast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01536_fuzz_cast.sql b/tests/queries/0_stateless/01536_fuzz_cast.sql new file mode 100644 index 00000000000..436d76b5c4c --- /dev/null +++ b/tests/queries/0_stateless/01536_fuzz_cast.sql @@ -0,0 +1 @@ +SELECT CAST(arrayJoin([NULL, '', '', NULL, '', NULL, '01.02.2017 03:04\005GMT', '', NULL, '01/02/2017 03:04:05 MSK01/02/\0017 03:04:05 MSK', '', NULL, '03/04/201903/04/201903/04/\001903/04/2019']), 'Enum8(\'a\' = 1, \'b\' = 2)') AS x; -- { serverError 349 } diff --git a/tests/queries/0_stateless/01537_fuzz_count_equal.reference b/tests/queries/0_stateless/01537_fuzz_count_equal.reference new file mode 100644 index 00000000000..dec7d2fabd2 --- /dev/null +++ b/tests/queries/0_stateless/01537_fuzz_count_equal.reference @@ -0,0 +1 @@ +\N diff --git a/tests/queries/0_stateless/01537_fuzz_count_equal.sql b/tests/queries/0_stateless/01537_fuzz_count_equal.sql new file mode 100644 index 00000000000..fde3fe19124 --- /dev/null +++ b/tests/queries/0_stateless/01537_fuzz_count_equal.sql @@ -0,0 +1 @@ +SELECT DISTINCT NULL = countEqual(materialize([arrayJoin([NULL, NULL, NULL]), NULL AS x, arrayJoin([255, 1025, NULL, NULL]), arrayJoin([2, 1048576, NULL, NULL])]), materialize(x)); diff --git a/tests/queries/0_stateless/01538_fuzz_aggregate.reference b/tests/queries/0_stateless/01538_fuzz_aggregate.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01538_fuzz_aggregate.sql b/tests/queries/0_stateless/01538_fuzz_aggregate.sql new file mode 100644 index 00000000000..13dadabda63 --- /dev/null +++ b/tests/queries/0_stateless/01538_fuzz_aggregate.sql @@ -0,0 +1,10 @@ +SELECT + count(), + sum(ns) +FROM +( + SELECT intDiv(number, NULL) AS k + FROM system.numbers_mt + GROUP BY k +) +ARRAY JOIN ns; -- { serverError 47 } diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.reference b/tests/queries/0_stateless/01542_dictionary_load_exception_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh new file mode 100755 index 00000000000..8782faac003 --- /dev/null +++ b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE database_for_dict" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS database_for_dict.table_for_dict" +$CLICKHOUSE_CLIENT --query "CREATE TABLE database_for_dict.table_for_dict (key_column UInt64, second_column UInt64, third_column String) ENGINE = MergeTree() ORDER BY key_column" +$CLICKHOUSE_CLIENT --query "INSERT INTO database_for_dict.table_for_dict VALUES (100500, 10000000, 'Hello world')" + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ordinary_db" +$CLICKHOUSE_CLIENT --query "CREATE DICTIONARY ordinary_db.dict1 ( key_column UInt64 DEFAULT 0, second_column UInt64 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(max_result_bytes=1)" + +function dict_get_thread() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + done +} + +export -f dict_get_thread; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 1fe199be48f..9249fbc0411 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + export CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:="test"} export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} -[ -n "$CLICKHOUSE_CONFIG_CLIENT" ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " -[ -n "${CLICKHOUSE_HOST}" ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " -[ -n "${CLICKHOUSE_PORT_TCP}" ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " -[ -n "${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}" ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " -[ -n "${CLICKHOUSE_DATABASE}" ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " +[ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " +[ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " +[ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " +[ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " +[ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} [ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} [ -x "$CLICKHOUSE_BINARY" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY client} export CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} -export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT0} ${CLICKHOUSE_CLIENT_OPT}"} +export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT0:-} ${CLICKHOUSE_CLIENT_OPT:-}"} [ -x "${CLICKHOUSE_BINARY}-local" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY} local"} export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} @@ -42,7 +44,7 @@ export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:="8443"} export CLICKHOUSE_PORT_HTTP_PROTO=${CLICKHOUSE_PORT_HTTP_PROTO:="http"} # Add database to url params -if [ -n "${CLICKHOUSE_URL_PARAMS}" ] +if [ -v CLICKHOUSE_URL_PARAMS ] then export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&database=${CLICKHOUSE_DATABASE}" else @@ -53,7 +55,7 @@ export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICK export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"} # Add url params to url -if [ -n "${CLICKHOUSE_URL_PARAMS}" ] +if [ -v CLICKHOUSE_URL_PARAMS ] then export CLICKHOUSE_URL="${CLICKHOUSE_URL}?${CLICKHOUSE_URL_PARAMS}" export CLICKHOUSE_URL_HTTPS="${CLICKHOUSE_URL_HTTPS}?${CLICKHOUSE_URL_PARAMS}" @@ -65,7 +67,7 @@ export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PO export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="10"} -export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} +export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q -s --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py index e50ac0a3f8b..06610f6e42b 100755 --- a/tests/testflows/aes_encryption/regression.py +++ b/tests/testflows/aes_encryption/regression.py @@ -48,12 +48,13 @@ xfails = { @TestFeature @Name("aes encryption") @ArgumentParser(argparser) +@Specifications(SRS_008_ClickHouse_AES_Encryption_Functions) @Requirements( RQ_SRS008_AES_Functions("1.0"), RQ_SRS008_AES_Functions_DifferentModes("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse AES encryption functions regression module. """ nodes = { diff --git a/tests/testflows/aes_encryption/requirements/requirements.py b/tests/testflows/aes_encryption/requirements/requirements.py index bae8b5cc3c1..d92e159547d 100644 --- a/tests/testflows/aes_encryption/requirements/requirements.py +++ b/tests/testflows/aes_encryption/requirements/requirements.py @@ -1,10 +1,1956 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200731.1222107. +# document by TestFlows v1.6.201026.1232822. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_008_ClickHouse_AES_Encryption_Functions = Specification( + name='SRS-008 ClickHouse AES Encryption Functions', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-008 ClickHouse AES Encryption Functions +# Software Requirements Specification + +## Table of Contents +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS008.AES.Functions](#rqsrs008aesfunctions) + * 4.1.2 [RQ.SRS008.AES.Functions.Compatability.MySQL](#rqsrs008aesfunctionscompatabilitymysql) + * 4.1.3 [RQ.SRS008.AES.Functions.Compatability.Dictionaries](#rqsrs008aesfunctionscompatabilitydictionaries) + * 4.1.4 [RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL](#rqsrs008aesfunctionscompatabilityenginedatabasemysql) + * 4.1.5 [RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL](#rqsrs008aesfunctionscompatabilityenginetablemysql) + * 4.1.6 [RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL](#rqsrs008aesfunctionscompatabilitytablefunctionmysql) + * 4.1.7 [RQ.SRS008.AES.Functions.DifferentModes](#rqsrs008aesfunctionsdifferentmodes) + * 4.1.8 [RQ.SRS008.AES.Functions.DataFromMultipleSources](#rqsrs008aesfunctionsdatafrommultiplesources) + * 4.1.9 [RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues](#rqsrs008aesfunctionssuppressoutputofsensitivevalues) + * 4.1.10 [RQ.SRS008.AES.Functions.InvalidParameters](#rqsrs008aesfunctionsinvalidparameters) + * 4.1.11 [RQ.SRS008.AES.Functions.MismatchedKey](#rqsrs008aesfunctionsmismatchedkey) + * 4.1.12 [RQ.SRS008.AES.Functions.Check.Performance](#rqsrs008aesfunctionscheckperformance) + * 4.1.13 [RQ.SRS008.AES.Function.Check.Performance.BestCase](#rqsrs008aesfunctioncheckperformancebestcase) + * 4.1.14 [RQ.SRS008.AES.Function.Check.Performance.WorstCase](#rqsrs008aesfunctioncheckperformanceworstcase) + * 4.1.15 [RQ.SRS008.AES.Functions.Check.Compression](#rqsrs008aesfunctionscheckcompression) + * 4.1.16 [RQ.SRS008.AES.Functions.Check.Compression.LowCardinality](#rqsrs008aesfunctionscheckcompressionlowcardinality) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS008.AES.Encrypt.Function](#rqsrs008aesencryptfunction) + * 4.2.2 [RQ.SRS008.AES.Encrypt.Function.Syntax](#rqsrs008aesencryptfunctionsyntax) + * 4.2.3 [RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors](#rqsrs008aesencryptfunctionnisttestvectors) + * 4.2.4 [RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText](#rqsrs008aesencryptfunctionparametersplaintext) + * 4.2.5 [RQ.SRS008.AES.Encrypt.Function.Parameters.Key](#rqsrs008aesencryptfunctionparameterskey) + * 4.2.6 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode](#rqsrs008aesencryptfunctionparametersmode) + * 4.2.7 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesencryptfunctionparametersmodevaluesformat) + * 4.2.8 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesencryptfunctionparametersmodevalueinvalid) + * 4.2.9 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.10 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.11 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.12 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.13 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.14 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.15 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.16 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.17 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.18 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.19 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.20 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.21 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.22 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.23 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.24 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.25 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.26 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.27 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.28 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.29 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.30 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.31 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.32 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.33 [RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesencryptfunctionparametersinitializationvector) + * 4.2.34 [RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesencryptfunctionparametersadditionalauthenticateddata) + * 4.2.35 [RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesencryptfunctionparametersreturnvalue) + * 4.2.36 [RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesencryptfunctionkeylengthinvalidlengtherror) + * 4.2.37 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesencryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.38 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesencryptfunctioninitializationvectornotvalidformode) + * 4.2.39 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesencryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.40 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesencryptfunctionadditionalauthenticationdatalength) + * 4.2.41 [RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.42 [RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.43 [RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.44 [RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.45 [RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.46 [RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.47 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.48 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.49 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.50 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.51 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.52 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.53 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.54 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.55 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.56 [RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.57 [RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.58 [RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.59 [RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.60 [RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.61 [RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.62 [RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.63 [RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.64 [RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.2.65 [RQ.SRS008.AES.Decrypt.Function](#rqsrs008aesdecryptfunction) + * 4.2.66 [RQ.SRS008.AES.Decrypt.Function.Syntax](#rqsrs008aesdecryptfunctionsyntax) + * 4.2.67 [RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText](#rqsrs008aesdecryptfunctionparametersciphertext) + * 4.2.68 [RQ.SRS008.AES.Decrypt.Function.Parameters.Key](#rqsrs008aesdecryptfunctionparameterskey) + * 4.2.69 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode](#rqsrs008aesdecryptfunctionparametersmode) + * 4.2.70 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesdecryptfunctionparametersmodevaluesformat) + * 4.2.71 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesdecryptfunctionparametersmodevalueinvalid) + * 4.2.72 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.73 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.74 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.75 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.76 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.77 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.78 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.79 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.80 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.81 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.82 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.83 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.84 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.85 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.86 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.87 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.88 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.89 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.90 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.91 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.92 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.93 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.94 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.95 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.96 [RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesdecryptfunctionparametersinitializationvector) + * 4.2.97 [RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesdecryptfunctionparametersadditionalauthenticateddata) + * 4.2.98 [RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesdecryptfunctionparametersreturnvalue) + * 4.2.99 [RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesdecryptfunctionkeylengthinvalidlengtherror) + * 4.2.100 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesdecryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.101 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesdecryptfunctioninitializationvectornotvalidformode) + * 4.2.102 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesdecryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.103 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesdecryptfunctionadditionalauthenticationdatalength) + * 4.2.104 [RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.105 [RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.106 [RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.107 [RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.108 [RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.109 [RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.110 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.111 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.112 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.113 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.114 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.115 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.116 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.117 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.118 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.119 [RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.120 [RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.121 [RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.122 [RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.123 [RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.124 [RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.125 [RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.126 [RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.127 [RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.3 [MySQL Specific Functions](#mysql-specific-functions) + * 4.3.1 [RQ.SRS008.AES.MySQL.Encrypt.Function](#rqsrs008aesmysqlencryptfunction) + * 4.3.2 [RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax](#rqsrs008aesmysqlencryptfunctionsyntax) + * 4.3.3 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText](#rqsrs008aesmysqlencryptfunctionparametersplaintext) + * 4.3.4 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key](#rqsrs008aesmysqlencryptfunctionparameterskey) + * 4.3.5 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode](#rqsrs008aesmysqlencryptfunctionparametersmode) + * 4.3.6 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesformat) + * 4.3.7 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqlencryptfunctionparametersmodevalueinvalid) + * 4.3.8 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.9 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.10 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.11 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.12 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.13 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.14 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.15 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.16 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.17 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.18 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.19 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.20 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.21 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.22 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.23 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.24 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.25 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.26 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.27 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.28 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.29 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.30 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.31 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.32 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqlencryptfunctionparametersinitializationvector) + * 4.3.33 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqlencryptfunctionparametersreturnvalue) + * 4.3.34 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqlencryptfunctionkeylengthtooshorterror) + * 4.3.35 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqlencryptfunctionkeylengthtoolong) + * 4.3.36 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.37 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtoolong) + * 4.3.38 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqlencryptfunctioninitializationvectornotvalidformode) + * 4.3.39 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.40 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.41 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.42 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.43 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.44 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.45 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.46 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.47 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.48 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.49 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.50 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.51 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.52 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.53 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.54 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.55 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.56 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.3.57 [RQ.SRS008.AES.MySQL.Decrypt.Function](#rqsrs008aesmysqldecryptfunction) + * 4.3.58 [RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax](#rqsrs008aesmysqldecryptfunctionsyntax) + * 4.3.59 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText](#rqsrs008aesmysqldecryptfunctionparametersciphertext) + * 4.3.60 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key](#rqsrs008aesmysqldecryptfunctionparameterskey) + * 4.3.61 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode](#rqsrs008aesmysqldecryptfunctionparametersmode) + * 4.3.62 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesformat) + * 4.3.63 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqldecryptfunctionparametersmodevalueinvalid) + * 4.3.64 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.65 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.66 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.67 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.68 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.69 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.70 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.71 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.72 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.73 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.74 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.75 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.76 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.77 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.78 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.79 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.80 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.81 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.82 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.83 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.84 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.85 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.86 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.87 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.88 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqldecryptfunctionparametersinitializationvector) + * 4.3.89 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqldecryptfunctionparametersreturnvalue) + * 4.3.90 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqldecryptfunctionkeylengthtooshorterror) + * 4.3.91 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqldecryptfunctionkeylengthtoolong) + * 4.3.92 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.93 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtoolong) + * 4.3.94 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqldecryptfunctioninitializationvectornotvalidformode) + * 4.3.95 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.96 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.97 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.98 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.99 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.100 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.101 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.102 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.103 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.104 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.105 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.106 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.107 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.108 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.109 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.110 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.111 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.112 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ofbkeyandinitializationvectorlength) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +Users need an ability to encrypt and decrypt column data with tenant specific keys. +Use cases include protection of sensitive column values and [GDPR] right to forget policies. +The implementation will support capabilities of the [MySQL aes_encrypt] and [MySQL aes_decrypt] +functions which encrypt and decrypt values using the [AES] (Advanced Encryption Standard) +algorithm. This functionality will enable encryption and decryption of data +accessed on remote [MySQL] servers via [MySQL Dictionary] or [MySQL Database Engine], +[MySQL Table Engine], or [MySQL Table Function]. + +## Terminology + +* **AES** - + Advanced Encryption Standard ([AES]) + +## Requirements + +### Generic + +#### RQ.SRS008.AES.Functions +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data. + +#### RQ.SRS008.AES.Functions.Compatability.MySQL +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7]. + +#### RQ.SRS008.AES.Functions.Compatability.Dictionaries +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed on remote +[MySQL] servers using [MySQL Dictionary]. + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine], + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine]. + +#### RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function]. + +#### RQ.SRS008.AES.Functions.DifferentModes +version: 1.0 + +[ClickHouse] SHALL allow different modes to be supported in a single SQL statement +using explicit function parameters. + +#### RQ.SRS008.AES.Functions.DataFromMultipleSources +version: 1.0 + +[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources +in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary], +[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function] +with possibly different encryption schemes. + +#### RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues +version: 1.0 + +[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log, +error log, and `query_log` table to prevent leakage of sensitive values. + +#### RQ.SRS008.AES.Functions.InvalidParameters +version: 1.0 + +[ClickHouse] SHALL return an error when parameters are invalid. + +#### RQ.SRS008.AES.Functions.Mismatched.Key +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched keys. + +#### RQ.SRS008.AES.Functions.Mismatched.IV +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.AAD +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.Mode +version: 1.0 + +[ClickHouse] SHALL return an error or garbage for mismatched mode. + +#### RQ.SRS008.AES.Functions.Check.Performance +version: 1.0 + +Performance of [AES] encryption functions SHALL be measured. + +#### RQ.SRS008.AES.Function.Check.Performance.BestCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the best case +scenario where there is one key, one initialization vector, and one large stream of data. + +#### RQ.SRS008.AES.Function.Check.Performance.WorstCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the worst case +where there are `N` keys, `N` initialization vectors and `N` very small streams of data. + +#### RQ.SRS008.AES.Functions.Check.Compression +version: 1.0 + +Effect of [AES] encryption on column compression SHALL be measured. + +#### RQ.SRS008.AES.Functions.Check.Compression.LowCardinality +version: 1.0 + +Effect of [AES] encryption on the compression of a column with [LowCardinality] data type +SHALL be measured. + +### Specific + +#### RQ.SRS008.AES.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt` function + +```sql +aes_encrypt(plaintext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors +version: 1.0 + +[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt` function + +```sql +aes_decrypt(ciphertext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as +the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +### MySQL Specific Functions + +#### RQ.SRS008.AES.MySQL.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function + +```sql +aes_encrypt_mysql(plaintext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_encrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function + +```sql +aes_decrypt_mysql(ciphertext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting any data type as +the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_decrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +## References + +* **GDPR:** https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +* **MySQL:** https://www.mysql.com/ +* **AES:** https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +* **ClickHouse:** https://clickhouse.tech +* **Git:** https://git-scm.com/ + +[OpenSSL]: https://www.openssl.org/ +[LowCardinality]: https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ +[MergeTree]: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/ +[MySQL Database Engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/ +[MySQL Table Engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/ +[MySQL Table Function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/ +[MySQL Dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql +[GCM]: https://en.wikipedia.org/wiki/Galois/Counter_Mode +[CTR]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR) +[CBC]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_block_chaining_(CBC) +[ECB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_(ECB) +[CFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB1]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB8]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB128]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[OFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Output_feedback_(OFB) +[GDPR]: https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +[RFC5116]: https://tools.ietf.org/html/rfc5116#section-5.1 +[MySQL]: https://www.mysql.com/ +[MySQL 5.7]: https://dev.mysql.com/doc/refman/5.7/en/ +[MySQL aes_encrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-encrypt +[MySQL aes_decrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-decrypt +[AES]: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +[ClickHouse]: https://clickhouse.tech +[GitHub repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/aes_encryption/requirements/requirements.md +[Revision history]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/aes_encryption/requirements/requirements.md +[Git]: https://git-scm.com/ +[NIST test vectors]: https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program +''') + RQ_SRS008_AES_Functions = Requirement( name='RQ.SRS008.AES.Functions', version='1.0', @@ -14,9 +1960,9 @@ RQ_SRS008_AES_Functions = Requirement( uid=None, description=( '[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.MySQL', @@ -27,9 +1973,9 @@ RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Dictionaries', @@ -41,9 +1987,9 @@ RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( description=( '[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n' '[MySQL] servers using [MySQL Dictionary].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL', @@ -54,9 +2000,9 @@ RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL', @@ -67,9 +2013,9 @@ RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL', @@ -80,9 +2026,9 @@ RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_DifferentModes = Requirement( name='RQ.SRS008.AES.Functions.DifferentModes', @@ -94,9 +2040,9 @@ RQ_SRS008_AES_Functions_DifferentModes = Requirement( description=( '[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n' 'using explicit function parameters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( name='RQ.SRS008.AES.Functions.DataFromMultipleSources', @@ -110,9 +2056,9 @@ RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( 'in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n' '[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n' 'with possibly different encryption schemes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', @@ -124,9 +2070,9 @@ RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( description=( '[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n' 'error log, and `query_log` table to prevent leakage of sensitive values.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_InvalidParameters = Requirement( name='RQ.SRS008.AES.Functions.InvalidParameters', @@ -137,9 +2083,9 @@ RQ_SRS008_AES_Functions_InvalidParameters = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error when parameters are invalid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.Key', @@ -150,9 +2096,9 @@ RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched keys.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.IV', @@ -163,9 +2109,9 @@ RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.AAD', @@ -176,9 +2122,9 @@ RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.Mode', @@ -189,9 +2135,9 @@ RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error or garbage for mismatched mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Performance = Requirement( name='RQ.SRS008.AES.Functions.Check.Performance', @@ -202,9 +2148,9 @@ RQ_SRS008_AES_Functions_Check_Performance = Requirement( uid=None, description=( 'Performance of [AES] encryption functions SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( name='RQ.SRS008.AES.Function.Check.Performance.BestCase', @@ -216,9 +2162,9 @@ RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( description=( 'Performance of [AES] encryption functions SHALL be checked for the best case\n' 'scenario where there is one key, one initialization vector, and one large stream of data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', @@ -230,9 +2176,9 @@ RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( description=( 'Performance of [AES] encryption functions SHALL be checked for the worst case\n' 'where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Compression = Requirement( name='RQ.SRS008.AES.Functions.Check.Compression', @@ -243,9 +2189,9 @@ RQ_SRS008_AES_Functions_Check_Compression = Requirement( uid=None, description=( 'Effect of [AES] encryption on column compression SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', @@ -257,9 +2203,9 @@ RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( description=( 'Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n' 'SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function = Requirement( name='RQ.SRS008.AES.Encrypt.Function', @@ -270,9 +2216,9 @@ RQ_SRS008_AES_Encrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Syntax', @@ -287,9 +2233,9 @@ RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( '```sql\n' 'aes_encrypt(plaintext, key, mode, [iv, aad])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', @@ -300,9 +2246,9 @@ RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( uid=None, description=( '[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', @@ -314,9 +2260,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( description=( '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' 'the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', @@ -328,9 +2274,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', @@ -342,9 +2288,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', @@ -360,9 +2306,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', @@ -375,9 +2321,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -389,9 +2335,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -403,9 +2349,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -417,9 +2363,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -431,9 +2377,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -445,9 +2391,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -459,9 +2405,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -473,9 +2419,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -487,9 +2433,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -501,9 +2447,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -515,9 +2461,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -529,9 +2475,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -543,9 +2489,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -557,9 +2503,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -571,9 +2517,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -585,9 +2531,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -599,9 +2545,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -613,9 +2559,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -627,9 +2573,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM', @@ -643,9 +2589,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM', @@ -659,9 +2605,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM', @@ -675,9 +2621,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR', @@ -689,9 +2635,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR', @@ -703,9 +2649,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR', @@ -717,9 +2663,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', @@ -732,9 +2678,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', @@ -747,9 +2693,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirem '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' 'parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data\n' 'for block modes that require it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', @@ -761,9 +2707,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the encrypted value of the data\n' 'using `String` data type as the result of `aes_encrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', @@ -774,9 +2720,9 @@ RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', @@ -787,9 +2733,9 @@ RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', @@ -800,9 +2746,9 @@ RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requiremen uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', @@ -813,9 +2759,9 @@ RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Re uid=None, description=( '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', @@ -826,9 +2772,9 @@ RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement uid=None, description=( '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -840,9 +2786,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -854,9 +2800,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -868,9 +2814,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -882,9 +2828,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -896,9 +2842,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -910,9 +2856,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -924,9 +2870,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -938,9 +2884,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -952,9 +2898,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -966,9 +2912,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' 'and if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -980,9 +2926,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -994,9 +2940,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -1008,9 +2954,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -1022,9 +2968,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -1036,9 +2982,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -1050,9 +2996,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -1064,9 +3010,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -1078,9 +3024,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', @@ -1092,9 +3038,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', @@ -1106,9 +3052,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', @@ -1120,9 +3066,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', @@ -1134,9 +3080,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', @@ -1148,9 +3094,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', @@ -1162,9 +3108,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function = Requirement( name='RQ.SRS008.AES.Decrypt.Function', @@ -1175,9 +3121,9 @@ RQ_SRS008_AES_Decrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Syntax', @@ -1192,9 +3138,9 @@ RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( '```sql\n' 'aes_decrypt(ciphertext, key, mode, [iv, aad])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', @@ -1206,9 +3152,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( description=( '[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n' 'the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', @@ -1220,9 +3166,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', @@ -1234,9 +3180,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', @@ -1252,9 +3198,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', @@ -1267,9 +3213,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -1281,9 +3227,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -1295,9 +3241,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -1309,9 +3255,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -1323,9 +3269,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -1337,9 +3283,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -1351,9 +3297,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -1365,9 +3311,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -1379,9 +3325,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -1393,9 +3339,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -1407,9 +3353,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -1421,9 +3367,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -1435,9 +3381,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -1449,9 +3395,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -1463,9 +3409,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -1477,9 +3423,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -1491,9 +3437,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -1505,9 +3451,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -1519,9 +3465,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM', @@ -1535,9 +3481,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM', @@ -1551,9 +3497,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM', @@ -1567,9 +3513,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR', @@ -1581,9 +3527,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR', @@ -1595,9 +3541,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR', @@ -1609,9 +3555,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', @@ -1624,9 +3570,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', @@ -1639,9 +3585,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirem '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' 'parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data\n' 'for block modes that require it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', @@ -1653,9 +3599,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the decrypted value of the data\n' 'using `String` data type as the result of `aes_decrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', @@ -1666,9 +3612,9 @@ RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', @@ -1679,9 +3625,9 @@ RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', @@ -1693,9 +3639,9 @@ RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requiremen description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function\n' 'for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', @@ -1707,9 +3653,9 @@ RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Re description=( '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function\n' 'for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', @@ -1720,9 +3666,9 @@ RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement uid=None, description=( '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -1734,9 +3680,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -1748,9 +3694,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -1762,9 +3708,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -1776,9 +3722,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -1790,9 +3736,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -1804,9 +3750,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -1818,9 +3764,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -1832,9 +3778,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -1846,9 +3792,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -1860,9 +3806,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' 'and if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -1874,9 +3820,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' 'or `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -1888,9 +3834,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -1902,9 +3848,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -1916,9 +3862,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -1930,9 +3876,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -1944,9 +3890,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -1958,9 +3904,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -1972,9 +3918,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', @@ -1986,9 +3932,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', @@ -2000,9 +3946,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', @@ -2014,9 +3960,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', @@ -2028,9 +3974,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', @@ -2042,9 +3988,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', @@ -2056,9 +4002,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function', @@ -2069,9 +4015,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', @@ -2086,9 +4032,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( '```sql\n' 'aes_encrypt_mysql(plaintext, key, mode, [iv])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', @@ -2100,9 +4046,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( description=( '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' 'the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', @@ -2114,9 +4060,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', @@ -2128,9 +4074,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', @@ -2145,9 +4091,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', @@ -2160,9 +4106,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -2174,9 +4120,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Require description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -2188,9 +4134,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Require description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -2202,9 +4148,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Require description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -2216,9 +4162,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Require description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -2230,9 +4176,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Require description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -2244,9 +4190,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Require description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -2258,9 +4204,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -2272,9 +4218,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -2286,9 +4232,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -2300,9 +4246,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -2314,9 +4260,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -2328,9 +4274,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -2342,9 +4288,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -2356,9 +4302,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -2370,9 +4316,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -2384,9 +4330,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Require description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -2398,9 +4344,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Require description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -2412,9 +4358,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Require description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', @@ -2426,9 +4372,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', @@ -2440,9 +4386,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', @@ -2454,9 +4400,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', @@ -2468,9 +4414,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', @@ -2482,9 +4428,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', @@ -2496,9 +4442,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', @@ -2511,9 +4457,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requireme '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', @@ -2525,9 +4471,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the encrypted value of the data\n' 'using `String` data type as the result of `aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', @@ -2539,9 +4485,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( description=( '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n' 'function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', @@ -2563,9 +4509,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' '\t return folded_key\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', @@ -2577,9 +4523,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' 'that is required for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', @@ -2591,9 +4537,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requi description=( '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' 'its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', @@ -2605,9 +4551,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requ description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n' 'function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -2619,9 +4565,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -2633,9 +4579,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -2647,9 +4593,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -2661,9 +4607,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -2675,9 +4621,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -2689,9 +4635,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -2703,9 +4649,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -2717,9 +4663,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -2731,9 +4677,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -2745,9 +4691,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' 'and if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -2759,9 +4705,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -2773,9 +4719,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -2787,9 +4733,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -2801,9 +4747,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -2815,9 +4761,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -2829,9 +4775,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -2843,9 +4789,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -2857,9 +4803,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function', @@ -2870,9 +4816,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', @@ -2887,9 +4833,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( '```sql\n' 'aes_decrypt_mysql(ciphertext, key, mode, [iv])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', @@ -2901,9 +4847,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( description=( '[ClickHouse] SHALL support `ciphertext` accepting any data type as\n' 'the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', @@ -2915,9 +4861,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', @@ -2929,9 +4875,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', @@ -2946,9 +4892,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', @@ -2961,9 +4907,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -2975,9 +4921,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Require description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -2989,9 +4935,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Require description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -3003,9 +4949,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Require description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -3017,9 +4963,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Require description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -3031,9 +4977,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Require description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -3045,9 +4991,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Require description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -3059,9 +5005,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -3073,9 +5019,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -3087,9 +5033,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -3101,9 +5047,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -3115,9 +5061,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -3129,9 +5075,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -3143,9 +5089,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -3157,9 +5103,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -3171,9 +5117,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -3185,9 +5131,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Require description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -3199,9 +5145,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Require description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -3213,9 +5159,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Require description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', @@ -3227,9 +5173,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', @@ -3241,9 +5187,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', @@ -3255,9 +5201,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', @@ -3269,9 +5215,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', @@ -3283,9 +5229,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', @@ -3297,9 +5243,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', @@ -3312,9 +5258,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requireme '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', @@ -3326,9 +5272,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the decrypted value of the data\n' 'using `String` data type as the result of `aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', @@ -3340,9 +5286,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( description=( '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n' 'function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', @@ -3364,9 +5310,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' '\t return folded_key\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', @@ -3378,9 +5324,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' 'that is required for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', @@ -3392,9 +5338,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requi description=( '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' 'its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', @@ -3406,9 +5352,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requ description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n' 'function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -3420,9 +5366,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -3434,9 +5380,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -3448,9 +5394,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -3462,9 +5408,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -3476,9 +5422,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -3490,9 +5436,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -3504,9 +5450,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -3518,9 +5464,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -3532,9 +5478,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -3546,9 +5492,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' 'and if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -3560,9 +5506,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -3574,9 +5520,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -3588,9 +5534,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -3602,9 +5548,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -3616,9 +5562,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -3630,9 +5576,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -3644,9 +5590,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -3658,6 +5604,6 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) diff --git a/tests/testflows/example/regression.py b/tests/testflows/example/regression.py index 2c0a778d39b..cb58b42ba4c 100755 --- a/tests/testflows/example/regression.py +++ b/tests/testflows/example/regression.py @@ -2,7 +2,7 @@ import sys from testflows.core import * -append_path(sys.path, "..") +append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser @@ -10,13 +10,13 @@ from helpers.argparser import argparser @TestFeature @Name("example") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """Simple example of how you can use TestFlows to test ClickHouse. """ nodes = { "clickhouse": ("clickhouse1",), } - + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster diff --git a/tests/testflows/helpers/argparser.py b/tests/testflows/helpers/argparser.py index 033c15a3bfe..03014becb76 100644 --- a/tests/testflows/helpers/argparser.py +++ b/tests/testflows/helpers/argparser.py @@ -1,5 +1,12 @@ import os +def onoff(v): + if v in ["yes", "1", "on"]: + return True + elif v in ["no", "0", "off"]: + return False + raise ValueError(f"invalid {v}") + def argparser(parser): """Default argument parser for regressions. """ @@ -10,4 +17,10 @@ def argparser(parser): parser.add_argument("--clickhouse-binary-path", type=str, dest="clickhouse_binary_path", help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path", - default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) \ No newline at end of file + default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) + + parser.add_argument("--stress", action="store_true", default=False, + help="enable stress testing (might take a long time)") + + parser.add_argument("--parallel", type=onoff, default=True, choices=["yes", "no", "on", "off", 0, 1], + help="enable parallelism for tests that support it") \ No newline at end of file diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py old mode 100644 new mode 100755 index 01ebfa8424b..d173547a916 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -7,6 +7,7 @@ import tempfile from testflows.core import * from testflows.asserts import error from testflows.connect import Shell +from testflows.uexpect import ExpectTimeoutError class QueryRuntimeException(Exception): """Exception during query execution on the server. @@ -25,7 +26,7 @@ class Node(object): def repr(self): return f"Node(name='{self.name}')" - def restart(self, timeout=120, safe=True): + def restart(self, timeout=300, safe=True): """Restart node. """ with self.cluster.lock: @@ -42,18 +43,46 @@ class Node(object): class ClickHouseNode(Node): """Node with ClickHouse server. """ - def wait_healthy(self, timeout=120): + def wait_healthy(self, timeout=300): with By(f"waiting until container {self.name} is healthy"): start_time = time.time() while True: - if self.query("select 1", no_checks=1, timeout=120, steps=False).exitcode == 0: + if self.query("select 1", no_checks=1, timeout=300, steps=False).exitcode == 0: break if time.time() - start_time < timeout: time.sleep(2) continue assert False, "container is not healthy" - def restart(self, timeout=120, safe=True, wait_healthy=True): + def stop(self, timeout=300, safe=True): + """Stop node. + """ + if safe: + self.query("SYSTEM STOP MOVES") + self.query("SYSTEM STOP MERGES") + self.query("SYSTEM FLUSH LOGS") + with By("waiting for 5 sec for moves and merges to stop"): + time.sleep(5) + with And("forcing to sync everything to disk"): + self.command("sync", timeout=30) + + with self.cluster.lock: + for key in list(self.cluster._bash.keys()): + if key.endswith(f"-{self.name}"): + shell = self.cluster._bash.pop(key) + shell.__exit__(None, None, None) + + self.cluster.command(None, f'{self.cluster.docker_compose} stop {self.name}', timeout=timeout) + + def start(self, timeout=300, wait_healthy=True): + """Start node. + """ + self.cluster.command(None, f'{self.cluster.docker_compose} start {self.name}', timeout=timeout) + + if wait_healthy: + self.wait_healthy(timeout) + + def restart(self, timeout=300, safe=True, wait_healthy=True): """Restart node. """ if safe: @@ -79,32 +108,43 @@ class ClickHouseNode(Node): def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, raise_on_exception=False, step=By, settings=None, *args, **kwargs): """Execute and check query. - :param sql: sql query :param message: expected message that should be in the output, default: None :param exitcode: expected exitcode, default: None """ + settings = list(settings or []) + + if hasattr(current().context, "default_query_settings"): + settings += current().context.default_query_settings + if len(sql) > 1024: with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: query.write(sql) query.flush() command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n" - for setting in settings or []: + for setting in settings: name, value = setting command += f" --{name} \"{value}\"" description = f""" echo -e \"{sql[:100]}...\" > {query.name} {command} """ - with step("executing command", description=description) if steps else NullStep(): - r = self.cluster.bash(None)(command, *args, **kwargs) + with step("executing command", description=description, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(None)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(None) else: command = f"echo -e \"{sql}\" | clickhouse client -n" - for setting in settings or []: + for setting in settings: name, value = setting command += f" --{name} \"{value}\"" - with step("executing command", description=command) if steps else NullStep(): - r = self.cluster.bash(self.name)(command, *args, **kwargs) + with step("executing command", description=command, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(self.name)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(self.name) + raise if no_checks: return r @@ -135,6 +175,7 @@ class Cluster(object): docker_compose="docker-compose", docker_compose_project_dir=None, docker_compose_file="docker-compose.yml"): + self.terminating = False self._bash = {} self.clickhouse_binary_path = clickhouse_binary_path self.configs_dir = configs_dir @@ -168,7 +209,7 @@ class Cluster(object): self.docker_compose += f" --no-ansi --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" self.lock = threading.Lock() - def shell(self, node, timeout=120): + def shell(self, node, timeout=300): """Returns unique shell terminal to be used. """ if node is None: @@ -181,14 +222,22 @@ class Cluster(object): shell.timeout = timeout return shell - def bash(self, node, timeout=120): + def bash(self, node, timeout=300): """Returns thread-local bash terminal to a specific node. - :param node: name of the service """ + test = current() + + if self.terminating: + if test and (test.cflags & MANDATORY): + pass + else: + raise InterruptedError("terminating") + current_thread = threading.current_thread() - id = f"{current_thread.ident}-{node}" + id = f"{current_thread.name}-{node}" + with self.lock: if self._bash.get(id) is None: if node is None: @@ -197,9 +246,30 @@ class Cluster(object): self._bash[id] = Shell(command=[ "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" ], name=node).__enter__() + self._bash[id].timeout = timeout + + # clean up any stale open shells for threads that have exited + active_thread_names = {thread.name for thread in threading.enumerate()} + + for bash_id in list(self._bash.keys()): + thread_name, node_name = bash_id.rsplit("-", 1) + if thread_name not in active_thread_names: + self._bash[bash_id].__exit__(None, None, None) + del self._bash[bash_id] + return self._bash[id] + def close_bash(self, node): + current_thread = threading.current_thread() + id = f"{current_thread.name}-{node}" + + with self.lock: + if self._bash.get(id) is None: + return + self._bash[id].__exit__(None, None, None) + del self._bash[id] + def __enter__(self): with Given("docker-compose cluster"): self.up() @@ -211,20 +281,21 @@ class Cluster(object): self.down() finally: with self.lock: - for shell in list(self._bash.values()): + for shell in self._bash.values(): shell.__exit__(type, value, traceback) def node(self, name): """Get object with node bound methods. - :param name: name of service name """ if name.startswith("clickhouse"): return ClickHouseNode(self, name) return Node(self, name) - def down(self, timeout=120): + def down(self, timeout=300): """Bring cluster down by executing docker-compose down.""" + self.terminating = True + try: bash = self.bash(None) with self.lock: @@ -236,7 +307,7 @@ class Cluster(object): else: self._bash[id] = shell finally: - return self.command(None, f"{self.docker_compose} down", timeout=timeout) + return self.command(None, f"{self.docker_compose} down --timeout 60", bash=bash, timeout=timeout) def up(self, timeout=30*60): if self.local: @@ -265,7 +336,7 @@ class Cluster(object): if cmd.exitcode != 0: continue with And("executing docker-compose down just in case it is up"): - cmd = self.command(None, f"{self.docker_compose} down 2>&1 | tee", exitcode=None, timeout=timeout) + cmd = self.command(None, f"{self.docker_compose} down --remove-orphans 2>&1 | tee", exitcode=None, timeout=timeout) if cmd.exitcode != 0: continue with And("executing docker-compose up"): @@ -286,18 +357,22 @@ class Cluster(object): for name in self.nodes["clickhouse"]: self.node(name).wait_healthy() - def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs): + def command(self, node, command, message=None, exitcode=None, steps=True, bash=None, *args, **kwargs): """Execute and check command. - :param node: name of the service :param command: command :param message: expected message that should be in the output, default: None :param exitcode: expected exitcode, default: None :param steps: don't break command into steps, default: True """ - debug(f"command() {node}, {command}") with By("executing command", description=command, format_description=False) if steps else NullStep(): - r = self.bash(node)(command, *args, **kwargs) + if bash is None: + bash = self.bash(node) + try: + r = bash(command, *args, **kwargs) + except ExpectTimeoutError: + self.close_bash(node) + raise if exitcode is not None: with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep(): assert r.exitcode == exitcode, error(r.output) diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index 9d0a5ca743f..ed75ce4fe75 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -33,7 +33,7 @@ xfails = { RQ_SRS_007_LDAP_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse integration with LDAP regression module. """ nodes = { @@ -43,6 +43,11 @@ def regression(self, local, clickhouse_binary_path): with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster + if stress is not None or not hasattr(self.context, "stress"): + self.context.stress = stress + if parallel is not None or not hasattr(self.context, "parallel"): + self.context.parallel = parallel + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.authentication.tests.multiple_servers", "scenario")) Feature(run=load("ldap.authentication.tests.connections", "feature")) diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index d322db70330..1c65a29fef4 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -460,14 +460,14 @@ time user configuration contains any of the `` entries. #### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined version: 1.0 -[ClickHouse] SHALL throw an error during any authentification attempt +[ClickHouse] SHALL throw an error during any authentication attempt if the name of the [LDAP] server used inside the `` entry is not defined in the `` section. #### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty version: 1.0 -[ClickHouse] SHALL throw an error during any authentification attempt +[ClickHouse] SHALL throw an error during any authentication attempt if the name of the [LDAP] server used inside the `` entry is empty. diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 967e097d1fa..60fbef9b8cd 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -1,10 +1,571 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200811.1124123. +# document by TestFlows v1.6.201026.1232822. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( + name='SRS-007 ClickHouse Authentication of Users via LDAP', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-007 ClickHouse Authentication of Users via LDAP + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) + * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) + * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) + * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) + * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) + * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) + * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) + * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) + * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) + * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) + * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) + * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) + * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) + * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) + * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) + * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) + * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) + * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) + * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) + * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) + * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) + * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) + * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) + * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) + * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) + * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) + * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) + * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) + * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) + * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) + * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) + * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) + * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) + * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) + * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) + * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) + * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) + * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) + * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) + * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) + * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) + * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) + * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) + * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) + * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) + * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) + * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) + * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.38 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.39 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +[ClickHouse] currently does not have any integration with [LDAP]. +As the initial step in integrating with [LDAP] this software requirements specification covers +only the requirements to enable authentication of users using an [LDAP] server. + +## Terminology + +* **CA** - + Certificate Authority ([CA]) + +* **LDAP** - + Lightweight Directory Access Protocol ([LDAP]) + +## Requirements + +### Generic + +#### RQ.SRS-007.LDAP.Authentication +version: 1.0 + +[ClickHouse] SHALL support user authentication via an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate +users. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. + +#### RQ.SRS-007.LDAP.Server.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +#### RQ.SRS-007.LDAP.User.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the user +has been deleted from the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications. + +### Specific + +#### RQ.SRS-007.LDAP.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. + +#### RQ.SRS-007.LDAP.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +#### RQ.SRS-007.LDAP.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.RBAC +version: 1.0 + +[ClickHouse] SHALL support creating users identified using an [LDAP] server using +the following RBAC command + +```sql +CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using +an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. + +```xml + + + + + my_ldap_server + + + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty +version: 1.0 + +[ClickHouse] SHALL not support empty string as a user name. + +#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP +version: 1.0 + +[ClickHouse] SHALL throw an error if `` is specified for the user and at the same +time user configuration contains any of the `` entries. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentication attempt +if the name of the [LDAP] server used inside the `` entry +is not defined in the `` section. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentication attempt +if the name of the [LDAP] server used inside the `` entry +is empty. + +#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer +version: 1.0 + +[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Long +version: 1.0 + +[ClickHouse] SHALL support long user names of at least 256 bytes +to specify users that can be authenticated using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 +version: 1.0 + +[ClickHouse] SHALL support user names that contain [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username. + +#### RQ.SRS-007.LDAP.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. + +#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users using an [LDAP] server. + +## References + +* **ClickHouse:** https://clickhouse.tech + +[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind +[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind +[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind +[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 +[OpenSSL]: https://www.openssl.org/ +[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html +[CA]: https://en.wikipedia.org/wiki/Certificate_authority +[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://github.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Git]: https://git-scm.com/ +''') + RQ_SRS_007_LDAP_Authentication = Requirement( name='RQ.SRS-007.LDAP.Authentication', version='1.0', @@ -14,9 +575,9 @@ RQ_SRS_007_LDAP_Authentication = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( name='RQ.SRS-007.LDAP.Authentication.MultipleServers', @@ -28,9 +589,9 @@ RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( description=( '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' 'users.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', @@ -41,9 +602,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', @@ -54,9 +615,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', @@ -68,9 +629,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( description=( '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' 'plain text `ldap://` protocol that is upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', @@ -81,9 +642,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( uid=None, description=( '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', @@ -94,9 +655,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( uid=None, description=( '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', @@ -107,9 +668,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = uid=None, description=( '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', @@ -120,9 +681,9 @@ RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.User.Configuration.Invalid', @@ -133,9 +694,9 @@ RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', @@ -147,9 +708,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', @@ -161,9 +722,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', @@ -175,9 +736,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( description=( '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Valid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Valid', @@ -189,9 +750,9 @@ RQ_SRS_007_LDAP_Authentication_Valid = Requirement( description=( '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' 'user name and password match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid', @@ -203,9 +764,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' 'do not match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', @@ -217,9 +778,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' 'has been deleted from the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', @@ -231,9 +792,9 @@ RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' 'on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', @@ -245,9 +806,9 @@ RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' 'for the user is changed on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', @@ -258,9 +819,9 @@ RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', @@ -271,9 +832,9 @@ RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel', @@ -284,9 +845,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( uid=None, description=( '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', @@ -299,9 +860,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( '[ClickHouse] SHALL support authentication of valid users and\n' 'prohibit authentication of invalid users using [LDAP] server\n' 'in parallel without having invalid attempts affecting valid authentications.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_UnreachableServer = Requirement( name='RQ.SRS-007.LDAP.UnreachableServer', @@ -312,9 +873,9 @@ RQ_SRS_007_LDAP_UnreachableServer = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Name', @@ -325,9 +886,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a server name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Host', @@ -339,9 +900,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port', @@ -352,9 +913,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', @@ -365,9 +926,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( uid=None, description=( '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', @@ -379,9 +940,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the prefix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', @@ -393,9 +954,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the suffix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', @@ -408,9 +969,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' '\n' "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', @@ -421,9 +982,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', @@ -435,9 +996,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( description=( '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' 'to enable SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', @@ -449,9 +1010,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( description=( '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' 'plain text `ldap://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', @@ -463,9 +1024,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( description=( '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' 'SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', @@ -477,9 +1038,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( description=( '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', @@ -491,9 +1052,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify\n' 'the minimum protocol version of SSL/TLS.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', @@ -505,9 +1066,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirem description=( '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' 'as a value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', @@ -518,9 +1079,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Require uid=None, description=( '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', @@ -532,9 +1093,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' 'certificate verification behavior.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', @@ -545,9 +1106,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requiremen uid=None, description=( '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', @@ -560,9 +1121,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' 'provided, the session SHALL be immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', @@ -576,9 +1137,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( 'enable requesting of client certificate. If no\n' 'certificate is provided, the session SHALL proceed normally.\n' 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', @@ -592,9 +1153,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( 'enable requesting of client certificate. If no certificate is provided, the session\n' 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' 'immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', @@ -606,9 +1167,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( description=( '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' 'disable requesting of client certificate.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', @@ -620,9 +1181,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' '[ClickHouse] to establish connection with the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', @@ -634,9 +1195,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' 'specified by the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', @@ -648,9 +1209,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify to a path to\n' 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', @@ -662,9 +1223,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', @@ -685,9 +1246,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( '\n' 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' '[ClickHouse] and therefore might change.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', @@ -718,9 +1279,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.RBAC', @@ -736,9 +1297,9 @@ RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( '```sql\n' "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Syntax', @@ -762,9 +1323,9 @@ RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', @@ -775,9 +1336,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a user name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', @@ -789,9 +1350,9 @@ RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( description=( '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' 'time user configuration contains any of the `` entries.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', @@ -801,12 +1362,12 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requireme type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' + '[ClickHouse] SHALL throw an error during any authentication attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is not defined in the `` section.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', @@ -816,12 +1377,12 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' + '[ClickHouse] SHALL throw an error during any authentication attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', @@ -832,9 +1393,9 @@ RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( uid=None, description=( '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', @@ -846,9 +1407,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( description=( '[ClickHouse] SHALL support long user names of at least 256 bytes\n' 'to specify users that can be authenticated using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', @@ -859,9 +1420,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Empty', @@ -872,9 +1433,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support authenticating users with empty username.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Long', @@ -885,9 +1446,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', @@ -898,9 +1459,9 @@ RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Empty', @@ -913,9 +1474,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( '[ClickHouse] SHALL not support authenticating users with empty passwords\n' 'even if an empty password is valid for the user and\n' 'is allowed by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Long', @@ -927,9 +1488,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( description=( '[ClickHouse] SHALL support long password of at least 256 bytes\n' 'that can be used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', @@ -941,6 +1502,6 @@ RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( description=( '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' 'used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 0fd9670fae0..ed8d46df92b 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -85,6 +85,8 @@ def add_config(config, timeout=60, restart=False): :param config: configuration file description :param timeout: timeout, default: 20 sec """ + node = current().context.node + def check_preprocessed_config_is_updated(after_removal=False): """Check that preprocessed config is updated. """ @@ -116,13 +118,16 @@ def add_config(config, timeout=60, restart=False): with When("I close terminal to the node to be restarted"): bash.close() - with And("I get the current log size"): - logsize = \ - node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ - 0].strip() + with And("I stop ClickHouse to apply the config changes"): + node.stop(safe=False) - with And("I restart ClickHouse to apply the config changes"): - node.restart(safe=False) + with And("I get the current log size"): + cmd = node.cluster.command(None, + f"stat --format=%s {os.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log") + logsize = cmd.output.split(" ")[0].strip() + + with And("I start ClickHouse back up"): + node.start() with Then("I tail the log file from using previous log size as the offset"): bash.prompt = bash.__class__.prompt @@ -139,7 +144,6 @@ def add_config(config, timeout=60, restart=False): f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) - node = current().context.node try: with Given(f"{config.name}"): if settings.debug: diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py index 6ce860a6fd2..bd404d54438 100755 --- a/tests/testflows/ldap/external_user_directory/regression.py +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -33,7 +33,7 @@ xfails = { RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse LDAP external user directory regression module. """ nodes = { @@ -42,6 +42,11 @@ def regression(self, local, clickhouse_binary_path): with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster + + if stress is not None or not hasattr(self.context, "stress"): + self.context.stress = stress + if parallel is not None or not hasattr(self.context, "parallel"): + self.context.parallel = parallel Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.external_user_directory.tests.simple", "scenario")) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 9b216e7dd30..47c10121b68 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -92,25 +92,23 @@ def parallel_login(self, server, user_count=10, timeout=200): with Given("a group of LDAP users"): users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*users): - tasks = [] - try: - with When("users try to login in parallel", description=""" - * with valid username and password - * with invalid username and valid password - * with valid username and invalid password - """): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + with ldap_users(*users): + tasks = [] + try: + with When("users try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( @@ -127,25 +125,23 @@ def parallel_login_with_the_same_user(self, server, timeout=200): with Given("only one LDAP user"): users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*users): - tasks = [] - try: - with When("the same user tries to login in parallel", description=""" - * with valid username and password - * with invalid username and valid password - * with valid username and invalid password - """): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + with ldap_users(*users): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario def login_after_ldap_external_user_directory_is_removed(self, server): @@ -162,6 +158,7 @@ def login_after_ldap_external_user_directory_is_removed(self, server): login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -204,6 +201,7 @@ def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=200 join(tasks, timeout) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -256,6 +254,7 @@ def parallel_login_with_multiple_servers(self, server, user_count=10, timeout=20 join(tasks, timeout) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -323,20 +322,18 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200): users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with rbac_users(*users): - tasks = [] - try: - with When("I login in parallel"): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + with rbac_users(*users): + tasks = [] + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( @@ -347,10 +344,8 @@ def login_after_user_is_added_to_ldap(self, server): """ user = {"cn": "myuser", "userpassword": "myuser"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with When(f"I add user to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server) + with When(f"I add user to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -363,27 +358,25 @@ def login_after_user_is_deleted_from_ldap(self, server): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I delete this user from LDAP"): - delete_user_from_ldap(user) + with When("I delete this user from LDAP"): + delete_user_from_ldap(user) - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -396,31 +389,29 @@ def login_after_user_password_changed_in_ldap(self, server): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") + with When("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) - with And("when I try to login with the new password it should work"): - login_and_execute_query(username=user["cn"], password="newpassword") + with And("when I try to login with the new password it should work"): + login_and_execute_query(username=user["cn"], password="newpassword") - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -434,27 +425,25 @@ def login_after_user_cn_changed_in_ldap(self, server): user = None new_user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I change user password in LDAP"): - new_user = change_user_cn_in_ldap(user, "myuser2") + with When("I change user password in LDAP"): + new_user = change_user_cn_in_ldap(user, "myuser2") - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if new_user is not None: - delete_user_from_ldap(new_user, exitcode=None) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) @TestScenario @Requirements( @@ -467,31 +456,29 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I restart LDAP server"): - self.context.ldap_node.restart() + with When("I restart LDAP server"): + self.context.ldap_node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -504,31 +491,29 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I restart ClickHouse server"): - self.context.node.restart() + with When("I restart ClickHouse server"): + self.context.node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -542,9 +527,7 @@ def valid_username_with_valid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -561,9 +544,7 @@ def valid_username_and_invalid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -575,10 +556,8 @@ def valid_username_and_password(self, server): username = "valid_username_and_password" user = {"cn": username, "userpassword": username} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with When(f"I add user {username} to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server) + with When(f"I add user {username} to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -593,9 +572,7 @@ def valid_username_and_password_invalid_server(self, server=None): exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) @TestScenario @Requirements( @@ -608,9 +585,7 @@ def valid_long_username_and_short_password(self, server): username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -626,9 +601,7 @@ def invalid_long_username_and_valid_short_password(self, server): exitcode = 4 message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -641,9 +614,7 @@ def valid_short_username_and_long_password(self, server): username = "long_password" user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -659,9 +630,7 @@ def valid_short_username_and_invalid_long_password(self, server): exitcode = 4 message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -677,9 +646,7 @@ def valid_username_and_invalid_password(self, server): exitcode = 4 message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -695,9 +662,7 @@ def invalid_username_and_valid_password(self, server): exitcode = 4 message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -710,9 +675,7 @@ def valid_utf8_username_and_ascii_password(self, server): username = "utf8_username_Gãńdåłf_Thê_Gręât" user = {"cn": username, "userpassword": "utf8_username"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -725,18 +688,14 @@ def valid_ascii_username_and_utf8_password(self, server): username = "utf8_password" user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario def empty_username_and_empty_password(self, server=None): """Check that we can login using empty username and empty password as it will use the default user and that has an empty password. """ - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - login_and_execute_query(username="", password="") + login_and_execute_query(username="", password="") @TestScenario @Requirements( @@ -763,18 +722,16 @@ def user_lookup_priority(self, server): "ldap": {"username": "ldap", "password": "userldap"} } - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): - with rbac_users({"cn": "local", "userpassword": "local"}): - with When("I try to login as 'default' user which is also defined in users.xml it should fail"): - login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): + with rbac_users({"cn": "local", "userpassword": "local"}): + with When("I try to login as 'default' user which is also defined in users.xml it should fail"): + login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) - with When("I try to login as 'local' user which is also defined in local storage it should fail"): - login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) + with When("I try to login as 'local' user which is also defined in local storage it should fail"): + login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) - with When("I try to login as 'ldap' user defined only in LDAP it should work"): - login_and_execute_query(**users["ldap"]) + with When("I try to login as 'ldap' user defined only in LDAP it should work"): + login_and_execute_query(**users["ldap"]) @TestOutline(Feature) @@ -795,5 +752,10 @@ def feature(self, servers=None, server=None, node="clickhouse1"): server = "openldap1" with ldap_servers(servers): - for scenario in loads(current_module(), Scenario): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + for scenario in loads(current_module(), Scenario, filter=~has.tag("custom config")): + Scenario(test=scenario, flags=TE)(server=server) + + for scenario in loads(current_module(), Scenario, filter=has.tag("custom config")): Scenario(test=scenario, flags=TE)(server=server) diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 0e9d06cf84a..9cc9aa85f93 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -9,7 +9,7 @@ from helpers.argparser import argparser @TestModule @Name("ldap") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, parallel=None, stress=None): """ClickHouse LDAP integration regression module. """ args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} @@ -18,4 +18,4 @@ def regression(self, local, clickhouse_binary_path): Feature(test=load("ldap.external_user_directory.regression", "regression"))(**args) if main(): - regression() + regression() \ No newline at end of file diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml index ada8eec5fc9..a7ed0d6e2b4 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml @@ -58,9 +58,44 @@ 9440 1 - - - + + + + + + clickhouse1 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + clickhouse2 @@ -73,8 +108,20 @@ 9000 - - + + + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + clickhouse1 @@ -94,6 +141,22 @@ + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml index ca65ffd5e04..768d2250b79 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml @@ -3,6 +3,7 @@ /etc/clickhouse-server/ssl/server.crt /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem none true diff --git a/tests/testflows/rbac/configs/clickhouse/config.xml b/tests/testflows/rbac/configs/clickhouse/config.xml index 65187edf806..4ec12232539 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.xml @@ -69,7 +69,7 @@ - + 0.0.0.0 /var/lib/clickhouse/access/ + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + users.xml @@ -160,7 +172,7 @@ - + @@ -220,7 +232,7 @@ See https://clickhouse.yandex/docs/en/table_engines/replication/ --> - + - +