diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4acdc293f..09ce72d20ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -409,7 +409,7 @@ ## ClickHouse release 20.6 -### ClickHouse release v20.6.3.28-stable +### ClickHouse release v20.6.3.28-stable #### New Feature @@ -2362,7 +2362,7 @@ No changes compared to v20.4.3.16-stable. * `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) * Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) * Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) +* Now table the first argument of `joinGet` function can be table identifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) * Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) * Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) * `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21cc74bbd2b..783a9f80b66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,25 +59,6 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# cmake 3.9+ needed. -# Usually impractical. -# See also ${ENABLE_THINLTO} -option(ENABLE_IPO "Full link time optimization") - -if(ENABLE_IPO) - cmake_policy(SET CMP0069 NEW) - include(CheckIPOSupported) - check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_NOT_SUPPORTED) - if(IPO_SUPPORTED) - message(STATUS "IPO/LTO is supported, enabling") - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>") - endif() -else() - message(STATUS "IPO/LTO not enabled.") -endif() - # Check that submodules are present only if source was downloaded with git if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") diff --git a/README.md b/README.md index 0c07021630b..03b5c988586 100644 --- a/README.md +++ b/README.md @@ -17,4 +17,6 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events -* [ClickHouse virtual office hours](https://www.eventbrite.com/e/clickhouse-october-virtual-meetup-office-hours-tickets-123129500651) on October 22, 2020. +* [The Second ClickHouse Meetup East (online)](https://www.eventbrite.com/e/the-second-clickhouse-meetup-east-tickets-126787955187) on October 31, 2020. +* [ClickHouse for Enterprise Meetup (online in Russian)](https://arenadata-events.timepad.ru/event/1465249/) on November 10, 2020. + diff --git a/base/common/StringRef.h b/base/common/StringRef.h index 4376876c077..b51b95456cb 100644 --- a/base/common/StringRef.h +++ b/base/common/StringRef.h @@ -51,7 +51,7 @@ struct StringRef }; /// Here constexpr doesn't implicate inline, see https://www.viva64.com/en/w/v1043/ -/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetics +/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetic /// and the UBSan thinks that something like nullptr + 8 is UB. constexpr const inline char empty_string_ref_addr{}; constexpr const inline StringRef EMPTY_STRING_REF{&empty_string_ref_addr, 0}; diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc index 3ac401cb108..6910164d0aa 100644 --- a/cmake/yandex/ya.make.versions.inc +++ b/cmake/yandex/ya.make.versions.inc @@ -11,11 +11,11 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR}) CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH}) CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\") -CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) -CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH}) -# TODO: not supported yet, not sure if ya.make supports arithmetics. +# TODO: not supported yet, not sure if ya.make supports arithmetic. CFLAGS (GLOBAL -DVERSION_INTEGER=0) CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\") diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index ab4857f0387..3aa5cb970db 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -192,7 +192,7 @@ set(SRCS ${HDFS3_SOURCE_DIR}/common/FileWrapper.h ) -# old kernels (< 3.17) doens't have SYS_getrandom. Always use POSIX implementation to have better compatibility +# old kernels (< 3.17) doesn't have SYS_getrandom. Always use POSIX implementation to have better compatibility set_source_files_properties(${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") # target diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index f5638e954a7..1485b0de3ea 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit f5638e954a79f50bac7c7a5deaa5a241e0ce8b5f +Subproject commit 1485b0de3eaa1508dfe49a5ba1e4aa2a71fd8335 diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 4e9259b214a..8fd89d60f85 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -31,10 +31,6 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ && chmod +x dpkg-deb \ && cp dpkg-deb /usr/bin -RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ - && dpkg -i /tmp/arrow-keyring.deb - # Libraries from OS are only needed to test the "unbundled" build (this is not used in production). RUN apt-get update \ && apt-get install \ diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index 604f187cc58..50671011a23 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -1,6 +1,10 @@ # docker build -t yandex/clickhouse-unbundled-builder . FROM yandex/clickhouse-deb-builder +RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ + && dpkg -i /tmp/arrow-keyring.deb + # Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get update \ && apt-get install \ diff --git a/docker/server/.dockerignore b/docker/server/.dockerignore new file mode 100644 index 00000000000..468a8cafb00 --- /dev/null +++ b/docker/server/.dockerignore @@ -0,0 +1,8 @@ +# post / preinstall scripts (not needed, we do it in Dockerfile) +alpine-root/install/* + +# docs (looks useless) +alpine-root/usr/share/doc/* + +# packages, etc. (used by prepare.sh) +alpine-root/tgz-packages/* \ No newline at end of file diff --git a/docker/server/.gitignore b/docker/server/.gitignore new file mode 100644 index 00000000000..4081b5f124c --- /dev/null +++ b/docker/server/.gitignore @@ -0,0 +1 @@ +alpine-root/* \ No newline at end of file diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine new file mode 100644 index 00000000000..fc2756eac8c --- /dev/null +++ b/docker/server/Dockerfile.alpine @@ -0,0 +1,26 @@ +FROM alpine + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 \ + TZ=UTC \ + CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml + +COPY alpine-root/ / + +# from https://github.com/ClickHouse/ClickHouse/blob/master/debian/clickhouse-server.postinst +RUN addgroup clickhouse \ + && adduser -S -H -h /nonexistent -s /bin/false -G clickhouse -g "ClickHouse server" clickhouse \ + && chown clickhouse:clickhouse /var/lib/clickhouse \ + && chmod 700 /var/lib/clickhouse \ + && chown root:clickhouse /var/log/clickhouse-server \ + && chmod 775 /var/log/clickhouse-server \ + && chmod +x /entrypoint.sh \ + && apk add --no-cache su-exec + +EXPOSE 9000 8123 9009 + +VOLUME /var/lib/clickhouse \ + /var/log/clickhouse-server + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh new file mode 100755 index 00000000000..30101225b3e --- /dev/null +++ b/docker/server/alpine-build.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -x + +REPO_CHANNEL="${REPO_CHANNEL:-stable}" # lts / testing / prestable / etc +REPO_URL="${REPO_URL:-"https://repo.yandex.ru/clickhouse/tgz/${REPO_CHANNEL}"}" +VERSION="${VERSION:-20.9.3.45}" + +# where original files live +DOCKER_BUILD_FOLDER="${BASH_SOURCE%/*}" + +# we will create root for our image here +CONTAINER_ROOT_FOLDER="${DOCKER_BUILD_FOLDER}/alpine-root" + +# where to put downloaded tgz +TGZ_PACKAGES_FOLDER="${CONTAINER_ROOT_FOLDER}/tgz-packages" + +# clean up the root from old runs +rm -rf "$CONTAINER_ROOT_FOLDER" + +mkdir -p "$TGZ_PACKAGES_FOLDER" + +PACKAGES=( "clickhouse-client" "clickhouse-server" "clickhouse-common-static" ) + +# download tars from the repo +for package in "${PACKAGES[@]}" +do + wget -q --show-progress "${REPO_URL}/${package}-${VERSION}.tgz" -O "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" +done + +# unpack tars +for package in "${PACKAGES[@]}" +do + tar xvzf "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" --strip-components=2 -C "$CONTAINER_ROOT_FOLDER" +done + +# prepare few more folders +mkdir -p "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/users.d" \ + "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d" \ + "${CONTAINER_ROOT_FOLDER}/var/log/clickhouse-server" \ + "${CONTAINER_ROOT_FOLDER}/var/lib/clickhouse" \ + "${CONTAINER_ROOT_FOLDER}/docker-entrypoint-initdb.d" \ + "${CONTAINER_ROOT_FOLDER}/lib64" + +cp "${DOCKER_BUILD_FOLDER}/docker_related_config.xml" "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d/" +cp "${DOCKER_BUILD_FOLDER}/entrypoint.alpine.sh" "${CONTAINER_ROOT_FOLDER}/entrypoint.sh" + +## get glibc components from ubuntu 20.04 and put them to expected place +docker pull ubuntu:20.04 +ubuntu20image=$(docker create --rm ubuntu:20.04) +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libc.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libdl.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libm.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" + +docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "yandex/clickhouse-server:${VERSION}-alpine" --pull \ No newline at end of file diff --git a/docker/server/entrypoint.alpine.sh b/docker/server/entrypoint.alpine.sh new file mode 100755 index 00000000000..e2edda9ca26 --- /dev/null +++ b/docker/server/entrypoint.alpine.sh @@ -0,0 +1,152 @@ +#!/bin/sh +#set -x + +DO_CHOWN=1 +if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then + DO_CHOWN=0 +fi + +CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" +CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + +# support --user +if [ "$(id -u)" = "0" ]; then + USER=$CLICKHOUSE_UID + GROUP=$CLICKHOUSE_GID + # busybox has setuidgid & chpst buildin + gosu="su-exec $USER:$GROUP" +else + USER="$(id -u)" + GROUP="$(id -g)" + gosu="" + DO_CHOWN=0 +fi + +# set some vars +CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" + +# port is needed to check if clickhouse-server is ready for connections +HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" + +# get CH directories locations +DATA_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=path || true)" +TMP_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=tmp_path || true)" +USER_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=user_files_path || true)" +LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.log || true)" +LOG_DIR="$(dirname $LOG_PATH || true)" +ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.errorlog || true)" +ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" +FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" + +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" +CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" +CLICKHOUSE_DB="${CLICKHOUSE_DB:-}" + +for dir in "$DATA_DIR" \ + "$ERROR_LOG_DIR" \ + "$LOG_DIR" \ + "$TMP_DIR" \ + "$USER_PATH" \ + "$FORMAT_SCHEMA_PATH" +do + # check if variable not empty + [ -z "$dir" ] && continue + # ensure directories exist + if ! mkdir -p "$dir"; then + echo "Couldn't create necessary directory: $dir" + exit 1 + fi + + if [ "$DO_CHOWN" = "1" ]; then + # ensure proper directories permissions + chown -R "$USER:$GROUP" "$dir" + elif [ "$(stat -c %u "$dir")" != "$USER" ]; then + echo "Necessary directory '$dir' isn't owned by user with id '$USER'" + exit 1 + fi +done + +# if clickhouse user is defined - create it (user "default" already exists out of box) +if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ]; then + echo "$0: create new user '$CLICKHOUSE_USER' instead 'default'" + cat < /etc/clickhouse-server/users.d/default-user.xml + + + + + + + + <${CLICKHOUSE_USER}> + default + + ::/0 + + ${CLICKHOUSE_PASSWORD} + default + + + +EOT +fi + +if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then + # Listen only on localhost until the initialization is done + $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- --listen_host=127.0.0.1 & + pid="$!" + + # check if clickhouse is ready to accept connections + # will try to send ping clickhouse via http_port (max 6 retries, with 1 sec timeout and 1 sec delay between retries) + tries=6 + while ! wget --spider -T 1 -q "http://localhost:$HTTP_PORT/ping" 2>/dev/null; do + if [ "$tries" -le "0" ]; then + echo >&2 'ClickHouse init process failed.' + exit 1 + fi + tries=$(( tries-1 )) + sleep 1 + done + + if [ ! -z "$CLICKHOUSE_PASSWORD" ]; then + printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" + fi + + clickhouseclient="clickhouse-client --multiquery -u $CLICKHOUSE_USER $WITH_PASSWORD " + + # create default database, if defined + if [ -n "$CLICKHOUSE_DB" ]; then + echo "$0: create database '$CLICKHOUSE_DB'" + "$clickhouseclient" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB"; + fi + + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; cat "$f" | "$clickhouseclient" ; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "$clickhouseclient"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'Finishing of ClickHouse init process failed.' + exit 1 + fi +fi + +# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments +if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then + exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" +fi + +# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image +exec "$@" diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 85a5dd866f8..8300c31681e 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -271,7 +271,7 @@ TESTS_TO_SKIP=( 00974_query_profiler # Look at DistributedFilesToInsert, so cannot run in parallel. - 01457_DistributedFilesToInsert + 01460_DistributedFilesToInsert 01541_max_memory_usage_for_user diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 258bc0a95f7..d0cf12baa9e 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -63,7 +63,7 @@ function configure # Make copies of the original db for both servers. Use hardlinks instead # of copying to save space. Before that, remove preprocessed configs and # system tables, because sharing them between servers with hardlinks may - # lead to weird effects. + # lead to weird effects. rm -r left/db ||: rm -r right/db ||: rm -r db0/preprocessed_configs ||: @@ -77,15 +77,12 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed - # Disable percpu arenas because they segfault when the process is bound to - # a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939 - # - # About the jemalloc settings: + # Change the jemalloc settings here. # https://github.com/jemalloc/jemalloc/wiki/Getting-Started - export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" + export MALLOC_CONF="confirm_conf:true" set -m # Spawn servers in their own process groups - + left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -211,7 +208,7 @@ function run_tests echo test "$test_name" # Don't profile if we're past the time limit. - # Use awk because bash doesn't support floating point arithmetics. + # Use awk because bash doesn't support floating point arithmetic. profile_seconds=$(awk "BEGIN { print ($profile_seconds_left > 0 ? 10 : 0) }") TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n") @@ -544,10 +541,10 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') as select abs(diff) > report_threshold and abs(diff) > stat_threshold as changed_fail, abs(diff) > report_threshold - 0.05 and abs(diff) > stat_threshold as changed_show, - + not changed_fail and stat_threshold > report_threshold + 0.10 as unstable_fail, not changed_show and stat_threshold > report_threshold - 0.05 as unstable_show, - + left, right, diff, stat_threshold, if(report_threshold > 0, report_threshold, 0.10) as report_threshold, query_metric_stats.test test, query_metric_stats.query_index query_index, @@ -770,7 +767,7 @@ create table all_tests_report engine File(TSV, 'report/all-queries.tsv') as -- The threshold for 2) is significantly larger than the threshold for 1), to -- avoid jitter. create view shortness - as select + as select (test, query_index) in (select * from file('analyze/marked-short-queries.tsv', TSV, 'test text, query_index int')) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index e69fdc0fce0..8793f3d22b4 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -17,14 +17,24 @@ service clickhouse-server start && sleep 5 if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then SKIP_LIST_OPT="--use-skip-list" fi -# We can have several additional options so we path them as array because it's -# more idiologically correct. -read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" function run_tests() { + # We can have several additional options so we path them as array because it's + # more idiologically correct. + read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" + + # Skip these tests, because they fail when we rerun them multiple times + if [ "$NUM_TRIES" -gt "1" ]; then + ADDITIONAL_OPTIONS+=('--skip') + ADDITIONAL_OPTIONS+=('00000_no_tests_to_skip') + fi + for i in $(seq 1 $NUM_TRIES); do - clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt + clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt + if [ ${PIPESTATUS[0]} -ne "0" ]; then + break; + fi done } diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 9565e39598c..06241d6d497 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.57 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.59 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md index dd14ee3b4b1..b0901ee6f6e 100644 --- a/docs/en/engines/table-engines/integrations/rabbitmq.md +++ b/docs/en/engines/table-engines/integrations/rabbitmq.md @@ -51,7 +51,7 @@ Optional parameters: - `rabbitmq_row_delimiter` – Delimiter character, which ends the message. - `rabbitmq_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. - `rabbitmq_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. -- `rabbitmq_num_queues` – The number of queues per consumer. Default: `1`. Specify more queues if the capacity of one queue per consumer is insufficient. +- `rabbitmq_num_queues` – Total number of queues. Default: `1`. Increasing this number can significantly improve performance. - `rabbitmq_queue_base` - Specify a hint for queue names. Use cases of this setting are described below. - `rabbitmq_deadletter_exchange` - Specify name for a [dead letter exchange](https://www.rabbitmq.com/dlx.html). You can create another table with this exchange name and collect messages in cases when they are republished to dead letter exchange. By default dead letter exchange is not specified. - `rabbitmq_persistent` - If set to 1 (true), in insert query delivery mode will be set to 2 (marks messages as 'persistent'). Default: `0`. @@ -148,4 +148,5 @@ Example: - `_channel_id` - ChannelID, on which consumer, who received the message, was declared. - `_delivery_tag` - DeliveryTag of the received message. Scoped per channel. - `_redelivered` - `redelivered` flag of the message. -- `_message_id` - MessageID of the received message; non-empty if was set, when message was published. +- `_message_id` - messageID of the received message; non-empty if was set, when message was published. +- `_timestamp` - timestamp of the received message; non-empty if was set, when message was published. diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md new file mode 100644 index 00000000000..45533d3733f --- /dev/null +++ b/docs/en/operations/opentelemetry.md @@ -0,0 +1,69 @@ +--- +toc_priority: 62 +toc_title: OpenTelemetry Support +--- + +# [experimental] OpenTelemetry Support + +[OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting +traces and metrics from distributed application. ClickHouse has some support +for OpenTelemetry. + +!!! warning "Warning" +This is an experimental feature that will change in backwards-incompatible ways in the future releases. + + +## Supplying Trace Context to ClickHouse + +ClickHouse accepts trace context HTTP headers, as described by +the [W3C recommendation](https://www.w3.org/TR/trace-context/). +It also accepts trace context over native protocol that is used for +communication between ClickHouse servers or between the client and server. +For manual testing, trace context headers conforming to the Trace Context +recommendation can be supplied to `clickhouse-client` using +`--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags. + +If no parent trace context is supplied, ClickHouse can start a new trace, with +probability controlled by the `opentelemetry_start_trace_probability` setting. + + +## Propagating the Trace Context + +The trace context is propagated to downstream services in the following cases: + +* Queries to remote ClickHouse servers, such as when using `Distributed` table + engine. + +* `URL` table function. Trace context information is sent in HTTP headers. + + +## Tracing the ClickHouse Itself + +ClickHouse creates _trace spans_ for each query and some of the query execution +stages, such as query planning or distributed queries. + +To be useful, the tracing information has to be exported to a monitoring system +that supports OpenTelemetry, such as Jaeger or Prometheus. ClickHouse avoids +a dependency on a particular monitoring system, instead only +providing the tracing data conforming to the standard. A natural way to do so +in an SQL RDBMS is a system table. OpenTelemetry trace span information +[required by the standard](https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/overview.md#span) +is stored in the system table called `system.opentelemetry_span_log`. + +The table must be enabled in the server configuration, see the `opentelemetry_span_log` +element in the default config file `config.xml`. It is enabled by default. + +The table has the following columns: + +- `trace_id` +- `span_id` +- `parent_span_id` +- `operation_name` +- `start_time` +- `finish_time` +- `finish_date` +- `attribute.name` +- `attribute.values` + +The tags or attributes are saved as two parallel arrays, containing the keys +and values. Use `ARRAY JOIN` to work with them. diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index 8a4b9ae7886..7c0c5b00c0d 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -80,4 +80,4 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## See Also {#see-also} - [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator -- [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 1145efbbc5f..2cc80dcffc1 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -551,7 +551,7 @@ formatReadableTimeDelta(column[, maximum_unit]) **Parameters** - `column` — A column with numeric time delta. -- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. +- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. Example: @@ -626,7 +626,12 @@ neighbor(column, offset[, default_value]) ``` The result of the function depends on the affected data blocks and the order of data in the block. -If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +!!! warning "Warning" + It can reach the neighbor rows only inside the currently processed data block. + +The rows order used during the calculation of `neighbor` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with ORDER BY and call the function from outside the subquery. **Parameters** @@ -731,8 +736,13 @@ Result: Calculates the difference between successive row values ​​in the data block. Returns 0 for the first row and the difference from the previous row for each subsequent row. +!!! warning "Warning" + It can reach the previos row only inside the currently processed data block. + The result of the function depends on the affected data blocks and the order of data in the block. -If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +The rows order used during the calculation of `runningDifference` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with ORDER BY and call the function from outside the subquery. Example: @@ -1584,7 +1594,7 @@ isDecimalOverflow(d, [p]) **Parameters** - `d` — value. [Decimal](../../sql-reference/data-types/decimal.md). -- `p` — precision. Optional. If omitted, the initial presicion of the first argument is used. Using of this paratemer could be helpful for data extraction to another DBMS or file. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `p` — precision. Optional. If omitted, the initial precision of the first argument is used. Using of this paratemer could be helpful for data extraction to another DBMS or file. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). **Returned values** diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index e5554b58e4a..262ae00dc95 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -169,7 +169,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **See Also** - [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type -- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions ## Logical Negation Operator {#logical-negation-operator} diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 35780856ef2..82326bf51cf 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -121,7 +121,7 @@ Defines storage time for values. Can be specified only for MergeTree-family tabl ## Column Compression Codecs {#codecs} -By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. +By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. You can also define the compression method for each individual column in the `CREATE TABLE` query. @@ -138,7 +138,7 @@ ENGINE = ... ``` -The `Default` codec can be specified to reference default compression which may dependend on different settings (and properties of data) in runtime. +The `Default` codec can be specified to reference default compression which may depend on different settings (and properties of data) in runtime. Example: `value UInt64 CODEC(Default)` — the same as lack of codec specification. Also you can remove current CODEC from the column and use default compression from config.xml: @@ -149,7 +149,7 @@ ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. -To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. +To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. !!! warning "Warning" You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index ef7b811e295..dedb5842d68 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -45,7 +45,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `rabbitmq_row_delimiter` – символ-разделитель, который завершает сообщение. - `rabbitmq_schema` – опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap’n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. - `rabbitmq_num_consumers` – количество потребителей на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. -- `rabbitmq_num_queues` – количество очередей на потребителя. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одной очереди на потребителя недостаточна. +- `rabbitmq_num_queues` – количество очередей. По умолчанию: `1`. Большее число очередей может сильно увеличить пропускную способность. - `rabbitmq_queue_base` - настройка для имен очередей. Сценарии использования описаны ниже. - `rabbitmq_persistent` - флаг, от которого зависит настройка 'durable' для сообщений при запросах `INSERT`. По умолчанию: `0`. - `rabbitmq_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `rabbitmq_skip_broken_messages = N`, то движок отбрасывает `N` сообщений, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. @@ -140,4 +140,5 @@ Example: - `_channel_id` - идентификатор канала `ChannelID`, на котором было получено сообщение. - `_delivery_tag` - значение `DeliveryTag` полученного сообщения. Уникально в рамках одного канала. - `_redelivered` - флаг `redelivered`. (Не равно нулю, если есть возможность, что сообщение было получено более, чем одним каналом.) -- `_message_id` - значение `MessageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. +- `_message_id` - значение поля `messageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. +- `_timestamp` - значение поля `timestamp` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3cd584c0e55..ace509d6691 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -218,6 +218,8 @@ private: QueryFuzzer fuzzer; int query_fuzzer_runs = 0; + std::optional suggest; + /// We will format query_id in interactive mode in various ways, the default is just to print Query id: ... std::vector> query_id_formats; @@ -577,10 +579,11 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); + suggest.emplace(); if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false)) { /// Load suggestion data from the server. - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); + suggest->load(connection_parameters, config().getInt("suggestion_limit")); } /// Load command history if present. @@ -607,7 +610,7 @@ private: highlight_callback = highlight; ReplxxLineReader lr( - Suggest::instance(), + *suggest, history_file, config().has("multiline"), query_extenders, @@ -615,7 +618,7 @@ private: highlight_callback); #elif defined(USE_READLINE) && USE_READLINE - ReadlineLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters); + ReadlineLineReader lr(*suggest, history_file, config().has("multiline"), query_extenders, query_delimiters); #else LineReader lr(history_file, config().has("multiline"), query_extenders, query_delimiters); #endif @@ -2324,6 +2327,8 @@ public: ("log-level", po::value(), "client log level") ("server_logs_file", po::value(), "put server logs into specified file") ("query-fuzzer-runs", po::value()->default_value(0), "query fuzzer runs") + ("opentelemetry-traceparent", po::value(), "OpenTelemetry traceparent header as described by W3C Trace Context recommendation") + ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ; Settings cmd_settings; @@ -2492,6 +2497,25 @@ public: ignore_error = true; } + if (options.count("opentelemetry-traceparent")) + { + std::string traceparent = options["opentelemetry-traceparent"].as(); + std::string error; + if (!context.getClientInfo().parseTraceparentHeader( + traceparent, error)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot parse OpenTelemetry traceparent '{}': {}", + traceparent, error); + } + } + + if (options.count("opentelemetry-tracestate")) + { + context.getClientInfo().opentelemetry_tracestate = + options["opentelemetry-tracestate"].as(); + } + argsToConfig(common_arguments, config(), 100); clearPasswordFromCommandLine(argc, argv); diff --git a/programs/client/Suggest.h b/programs/client/Suggest.h index b13289ac322..03332088cbe 100644 --- a/programs/client/Suggest.h +++ b/programs/client/Suggest.h @@ -18,10 +18,11 @@ namespace ErrorCodes class Suggest : public LineReader::Suggest, boost::noncopyable { public: - static Suggest & instance() + Suggest(); + ~Suggest() { - static Suggest instance; - return instance; + if (loading_thread.joinable()) + loading_thread.join(); } void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit); @@ -30,12 +31,6 @@ public: static constexpr int MIN_SERVER_REVISION = 54406; private: - Suggest(); - ~Suggest() - { - if (loading_thread.joinable()) - loading_thread.join(); - } void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit); void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query); diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 2d019f81367..b9dde555788 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include @@ -47,9 +48,9 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; extern const int SYNTAX_ERROR; extern const int CANNOT_LOAD_CONFIG; + extern const int FILE_ALREADY_EXISTS; } @@ -121,31 +122,43 @@ void LocalServer::tryInitPath() } else { - // Default unique path in the system temporary directory. - const auto tmp = std::filesystem::temp_directory_path(); - const auto default_path = tmp - / fmt::format("clickhouse-local-{}", getpid()); + // The path is not provided explicitly - use a unique path in the system temporary directory + // (or in the current dir if temporary don't exist) + Poco::Logger * log = &logger(); + std::filesystem::path parent_folder; + std::filesystem::path default_path; + + try + { + // try to guess a tmp folder name, and check if it's a directory (throw exception otherwise) + parent_folder = std::filesystem::temp_directory_path(); + + } + catch (const std::filesystem::filesystem_error& e) + { + // tmp folder don't exists? misconfiguration? chroot? + LOG_DEBUG(log, "Can not get temporary folder: {}", e.what()); + parent_folder = std::filesystem::current_path(); + + std::filesystem::is_directory(parent_folder); // that will throw an exception if it's not a directory + LOG_DEBUG(log, "Will create working directory inside current directory: {}", parent_folder.string()); + } + + /// we can have another clickhouse-local running simultaneously, even with the same PID (for ex. - several dockers mounting the same folder) + /// or it can be some leftovers from other clickhouse-local runs + /// as we can't accurately distinguish those situations we don't touch any existent folders + /// we just try to pick some free name for our working folder + + default_path = parent_folder / fmt::format("clickhouse-local-{}-{}-{}", getpid(), time(nullptr), randomSeed()); if (exists(default_path)) - { - // This is a directory that is left by a previous run of - // clickhouse-local that had the same pid and did not complete - // correctly. Remove it, with an additional sanity check. - if (!std::filesystem::equivalent(default_path.parent_path(), tmp)) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "The temporary directory of clickhouse-local '{}' is not" - " inside the system temporary directory '{}'. Will not delete" - " it", default_path.string(), tmp.string()); - } - - remove_all(default_path); - } + throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Unsuccessful attempt to create working directory: {} exist!", default_path.string()); create_directory(default_path); temporary_directory_to_delete = default_path; path = default_path.string(); + LOG_DEBUG(log, "Working directory created: {}", path); } if (path.back() != '/') @@ -438,23 +451,12 @@ void LocalServer::setupUsers() void LocalServer::cleanup() { - // Delete the temporary directory if needed. Just in case, check that it is - // in the system temporary directory, not to delete user data if there is a - // bug. + // Delete the temporary directory if needed. if (temporary_directory_to_delete) { - const auto tmp = std::filesystem::temp_directory_path(); const auto dir = *temporary_directory_to_delete; temporary_directory_to_delete.reset(); - - if (!std::filesystem::equivalent(dir.parent_path(), tmp)) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "The temporary directory of clickhouse-local '{}' is not inside" - " the system temporary directory '{}'. Will not delete it", - dir.string(), tmp.string()); - } - + LOG_DEBUG(&logger(), "Removing temporary directory: {}", dir.string()); remove_all(dir); } } diff --git a/programs/server/config.xml b/programs/server/config.xml index 5bdec5377fd..e17b59671af 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -270,7 +270,7 @@ This parameter is mandatory and cannot be empty. roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. - If any of the listed roles is not defined locally at the time of authentication, the authenthication attept + If any of the listed roles is not defined locally at the time of authentication, the authenthication attempt will fail as if the provided password was incorrect. Example: @@ -628,6 +628,31 @@ 60000 + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + + + diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 61ab4c8002d..0459022cb1a 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -100,7 +100,7 @@ namespace if (res & alter_table) res |= alter_view; - /// CREATE TABLE (on any database/table) => CREATE_TEMPORARY_TABLE (global) + /// CREATE TABLE (on any database/table) => CREATE_TEMPORARY_TABLE (global) static const AccessFlags create_temporary_table = AccessType::CREATE_TEMPORARY_TABLE; if ((level == 0) && (max_flags_with_children & create_table)) res |= create_temporary_table; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8dd219e07d7..58821e7de4b 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -585,7 +585,7 @@ void IAccessStorage::throwInvalidPassword() void IAccessStorage::throwCannotAuthenticate(const String & user_name) { - /// We use the same message for all authentification failures because we don't want to give away any unnecessary information for security reasons, + /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons, /// only the log will show the exact reason. throw Exception(user_name + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED); } diff --git a/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp b/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp index e338b060b12..6f4f254ae8f 100644 --- a/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp +++ b/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp @@ -31,7 +31,7 @@ AggregateFunctionPtr createAggregateFunctionRate(const std::string & name, const void registerAggregateFunctionRate(AggregateFunctionFactory & factory) { - factory.registerFunction("boundingRatio", createAggregateFunctionRate, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("boundingRatio", createAggregateFunctionRate); } } diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.h b/src/AggregateFunctions/AggregateFunctionGroupArray.h index 83e096c797b..02b9003eb96 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h @@ -296,7 +296,7 @@ public: { typename ColumnVector::Container & data_to = assert_cast &>(arr_to.getData()).getData(); if constexpr (is_big_int_v) - // is data_to empty? we should probaly use std::vector::insert then + // is data_to empty? we should probably use std::vector::insert then for (auto it = this->data(place).value.begin(); it != this->data(place).value.end(); it++) data_to.push_back(*it); else diff --git a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp index 20472279dba..796ff028424 100644 --- a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp +++ b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp @@ -45,7 +45,7 @@ AggregateFunctionPtr createAggregateFunctionRankCorrelation(const std::string & void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory & factory) { - factory.registerFunction("rankCorr", createAggregateFunctionRankCorrelation, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("rankCorr", createAggregateFunctionRankCorrelation); } } diff --git a/src/AggregateFunctions/AggregateFunctionRetention.cpp b/src/AggregateFunctions/AggregateFunctionRetention.cpp index 4497703c550..c9d475c78f3 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.cpp +++ b/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -32,7 +32,7 @@ AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, void registerAggregateFunctionRetention(AggregateFunctionFactory & factory) { - factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("retention", createAggregateFunctionRetention); } } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index a2c36e43488..58fc9e5b5b9 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -47,6 +47,6 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & nam void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index d260a6be980..0aef8f3ee2a 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -20,9 +20,17 @@ namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } +#if defined(OS_DARWIN) +extern "C" +{ + double lgammal_r(double x, int * signgamp); +} +#endif + + namespace DB { @@ -98,7 +106,7 @@ struct AggregateFunctionStudentTTestData final Float64 getSSquared() const { - /// The original formulae looks like + /// The original formulae looks like /// \frac{\sum_{i = 1}^{n_x}{(x_i - \bar{x}) ^ 2} + \sum_{i = 1}^{n_y}{(y_i - \bar{y}) ^ 2}}{n_x + n_y - 2} /// But we made some mathematical transformations not to store original sequences. /// Also we dropped sqrt, because later it will be squared later. @@ -150,7 +158,8 @@ struct AggregateFunctionStudentTTestData final const Float64 t = getTStatisticSquared(); auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); - Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + int unused; + Float64 denominator = std::exp(lgammal_r(v / 2, &unused) + lgammal_r(0.5, &unused) - lgammal_r(v / 2 + 0.5, &unused)); return numenator / denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp index cd0599729b9..c8711c257f8 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp @@ -28,8 +28,8 @@ namespace void registerAggregateFunctionTimeSeriesGroupSum(AggregateFunctionFactory & factory) { - factory.registerFunction("timeSeriesGroupSum", createAggregateFunctionTimeSeriesGroupSum, AggregateFunctionFactory::CaseInsensitive); - factory.registerFunction("timeSeriesGroupRateSum", createAggregateFunctionTimeSeriesGroupSum, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("timeSeriesGroupSum", createAggregateFunctionTimeSeriesGroupSum); + factory.registerFunction("timeSeriesGroupRateSum", createAggregateFunctionTimeSeriesGroupSum); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 483c99dde9b..0dcb125305d 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -44,6 +44,6 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 175e0171606..b598f25162e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -18,11 +18,20 @@ #include + namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } +#if defined(OS_DARWIN) +extern "C" +{ + double lgammal_r(double x, int * signgamp); +} +#endif + + namespace DB { @@ -159,9 +168,10 @@ struct AggregateFunctionWelchTTestData final { const Float64 v = getDegreesOfFreedom(); const Float64 t = getTStatisticSquared(); - auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; + auto f = [&v] (double x) { return std::pow(x, v / 2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); - Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + int unused; + Float64 denominator = std::exp(lgammal_r(v / 2, &unused) + lgammal_r(0.5, &unused) - lgammal_r(v / 2 + 0.5, &unused)); return numenator / denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index 872c70c2b98..1e9f2782d95 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -58,7 +58,7 @@ AggregateFunctionPtr createAggregateFunctionWindowFunnel(const std::string & nam void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory & factory) { - factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel); } } diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index 3097070c651..eae24c1f3e9 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -39,8 +39,8 @@ namespace ErrorCodes namespace detail { -const size_t DEFAULT_SAMPLE_COUNT = 8192; -const auto MAX_SKIP_DEGREE = sizeof(UInt32) * 8; + const size_t DEFAULT_MAX_SAMPLE_SIZE = 8192; + const auto MAX_SKIP_DEGREE = sizeof(UInt32) * 8; } /// What if there is not a single value - throw an exception, or return 0 or NaN in the case of double? @@ -50,6 +50,7 @@ enum class ReservoirSamplerDeterministicOnEmpty RETURN_NAN_OR_ZERO, }; + template class ReservoirSamplerDeterministic @@ -60,8 +61,8 @@ class ReservoirSamplerDeterministic } public: - ReservoirSamplerDeterministic(const size_t sample_count_ = DEFAULT_SAMPLE_COUNT) - : sample_count{sample_count_} + ReservoirSamplerDeterministic(const size_t max_sample_size_ = detail::DEFAULT_MAX_SAMPLE_SIZE) + : max_sample_size{max_sample_size_} { } @@ -131,8 +132,8 @@ public: void merge(const ReservoirSamplerDeterministic & b) { - if (sample_count != b.sample_count) - throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different sample_count"); + if (max_sample_size != b.max_sample_size) + throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different max sample size"); sorted = false; if (b.skip_degree > skip_degree) @@ -150,11 +151,16 @@ public: void read(DB::ReadBuffer & buf) { - DB::readIntBinary(sample_count, buf); + size_t size = 0; + DB::readIntBinary(size, buf); DB::readIntBinary(total_values, buf); - samples.resize(std::min(total_values, sample_count)); - for (size_t i = 0; i < samples.size(); ++i) + /// Compatibility with old versions. + if (size > total_values) + size = total_values; + + samples.resize(size); + for (size_t i = 0; i < size; ++i) DB::readPODBinary(samples[i], buf); sorted = false; @@ -162,10 +168,11 @@ public: void write(DB::WriteBuffer & buf) const { - DB::writeIntBinary(sample_count, buf); + size_t size = samples.size(); + DB::writeIntBinary(size, buf); DB::writeIntBinary(total_values, buf); - for (size_t i = 0; i < std::min(sample_count, total_values); ++i) + for (size_t i = 0; i < size; ++i) DB::writePODBinary(samples[i], buf); } @@ -174,18 +181,19 @@ private: using Element = std::pair; using Array = DB::PODArray; - size_t sample_count; - size_t total_values{}; - bool sorted{}; + const size_t max_sample_size; /// Maximum amount of stored values. + size_t total_values = 0; /// How many values were inserted (regardless if they remain in sample or not). + bool sorted = false; Array samples; - UInt8 skip_degree{}; + UInt8 skip_degree = 0; /// The number N determining that we save only one per 2^N elements in average. void insertImpl(const T & v, const UInt32 hash) { - /// @todo why + 1? I don't quite recall - while (samples.size() + 1 >= sample_count) + /// Make a room for plus one element. + while (samples.size() >= max_sample_size) { - if (++skip_degree > detail::MAX_SKIP_DEGREE) + ++skip_degree; + if (skip_degree > detail::MAX_SKIP_DEGREE) throw DB::Exception{"skip_degree exceeds maximum value", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED}; thinOut(); } @@ -195,35 +203,17 @@ private: void thinOut() { - auto size = samples.size(); - for (size_t i = 0; i < size;) - { - if (!good(samples[i].second)) - { - /// swap current element with the last one - std::swap(samples[size - 1], samples[i]); - --size; - } - else - ++i; - } - - if (size != samples.size()) - { - samples.resize(size); - sorted = false; - } + samples.resize(std::distance(samples.begin(), + std::remove_if(samples.begin(), samples.end(), [this](const auto & elem){ return !good(elem.second); }))); + sorted = false; } void sortIfNeeded() { if (sorted) return; + std::sort(samples.begin(), samples.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; }); sorted = true; - std::sort(samples.begin(), samples.end(), [] (const std::pair & lhs, const std::pair & rhs) - { - return lhs.first < rhs.first; - }); } template diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index b7fb22aeb0e..550a44a23a2 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -6,6 +6,8 @@ #include #include +#include + #if defined(MEMORY_SANITIZER) #include #endif diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index bdbc941c1e7..51248a598af 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -344,7 +344,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Shift all NULL values to the end. for (const auto & [first, last] : equal_ranges) { - /// Current interval is righter than limit. + /// Current interval is righter than limit. if (limit && first > limit) break; diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index 99facea6055..99e134675f6 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -82,7 +82,7 @@ public: * @see DB::ColumnUnique * * The most common example uses https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ columns. - * Consider data type @e LC(String). The inner type here is @e String which is more or less a contigous memory + * Consider data type @e LC(String). The inner type here is @e String which is more or less a contiguous memory * region, so it can be easily represented as a @e StringRef. So we pass that ref to this function and get its * index in the dictionary, which can be used to operate with the indices column. */ diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index ef0b82666dd..3d6a2d6f99c 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -54,6 +54,7 @@ M(LocalThread, "Number of threads in local thread pools. Should be similar to GlobalThreadActive.") \ M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ M(DistributedFilesToInsert, "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.") \ + M(TablesToDropQueueSize, "Number of dropped tables, that are waiting for background data removal.") \ namespace CurrentMetrics { diff --git a/src/Common/FileSyncGuard.h b/src/Common/FileSyncGuard.h index 6451f6ebf36..486b02d0f24 100644 --- a/src/Common/FileSyncGuard.h +++ b/src/Common/FileSyncGuard.h @@ -5,15 +5,15 @@ namespace DB { -/// Helper class, that recieves file descriptor and does fsync for it in destructor. +/// Helper class, that receives file descriptor and does fsync for it in destructor. /// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end. /// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version. /// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496 class FileSyncGuard { public: - /// NOTE: If you have already opened descriptor, it's preffered to use - /// this constructor instead of construnctor with path. + /// NOTE: If you have already opened descriptor, it's preferred to use + /// this constructor instead of constructor with path. FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {} FileSyncGuard(const DiskPtr & disk_, const String & path) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 21116e9d432..e527e97d608 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -234,13 +234,13 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::is_same_v ? CurrentMetrics::GlobalThreadActive : CurrentMetrics::LocalThreadActive); job(); - /// job should be reseted before decrementing scheduled_jobs to + /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; } catch (...) { - /// job should be reseted before decrementing scheduled_jobs to + /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index fc5318b11fb..d10d5981d57 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -152,7 +152,7 @@ void TraceCollector::run() if (trace_log) { // time and time_in_microseconds are both being constructed from the same timespec so that the - // times will be equal upto the precision of a second. + // times will be equal up to the precision of a second. struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index abb8158781b..f5c57781eef 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1288,13 +1288,13 @@ void ZooKeeper::receiveEvent() response->removeRootPath(root_path); } - /// Instead of setting the watch in sendEvent, set it in receiveEvent becuase need to check the response. + /// Instead of setting the watch in sendEvent, set it in receiveEvent because need to check the response. /// The watch shouldn't be set if the node does not exist and it will never exist like sequential ephemeral nodes. /// By using getData() instead of exists(), a watch won't be set if the node doesn't exist. if (request_info.watch) { bool add_watch = false; - /// 3 indicates the ZooKeeperExistsRequest. + /// 3 indicates the ZooKeeperExistsRequest. // For exists, we set the watch on both node exist and nonexist case. // For other case like getData, we only set the watch when node exists. if (request_info.request->getOpNum() == 3) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index be2f697e1b3..7a6b605d015 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -185,9 +185,9 @@ void CompressedReadBufferBase::decompress(char * to, size_t size_decompressed, s } else { - throw Exception("Data compressed with different methods, given method byte " + throw Exception("Data compressed with different methods, given method byte 0x" + getHexUIntLowercase(method) - + ", previous method byte " + + ", previous method byte 0x" + getHexUIntLowercase(codec->getMethodByte()), ErrorCodes::CANNOT_DECOMPRESS); } diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index fe6a5b2dacd..46d7d7dfcc4 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -87,7 +87,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr else throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - /// Default codec replaced with current default codec which may dependend on different + /// Default codec replaced with current default codec which may depend on different /// settings (and properties of data) in runtime. CompressionCodecPtr result_codec; if (codec_family_name == DEFAULT_CODEC_NAME) diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index baf6e9b2b86..3746753df8b 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -26,7 +26,7 @@ void ICompressionCodec::setCodecDescription(const String & codec_name, const AST std::shared_ptr result = std::make_shared(); result->name = "CODEC"; - /// Special case for codec Multiple, which doens't have name. It's just list + /// Special case for codec Multiple, which doesn't have name. It's just list /// of other codecs. if (codec_name.empty()) { diff --git a/src/Core/Defines.h b/src/Core/Defines.h index ba3d37242fa..4d7d8e08ac3 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -67,11 +67,14 @@ /// Minimum revision supporting SettingsBinaryFormat::STRINGS. #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 +/// Minimum revision supporting OpenTelemetry +#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY 54442 + /// Mininum revision supporting interserver secret. #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 /// Version of ClickHouse TCP protocol. Increment it manually when you change the protocol. -#define DBMS_TCP_PROTOCOL_VERSION 54441 +#define DBMS_TCP_PROTOCOL_VERSION 54442 /// The boundary on which the blocks for asynchronous file operations should be aligned. #define DEFAULT_AIO_FILE_BLOCK_SIZE 4096 diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1179c0eb46b..c09c4b3b034 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -705,7 +705,7 @@ namespace MySQLReplication break; } default: - throw ReplicationError("Position update with unsupport event", ErrorCodes::LOGICAL_ERROR); + throw ReplicationError("Position update with unsupported event", ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b94883ca871..4e95d1ab395 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -222,6 +222,7 @@ class IColumn; M(UInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \ + M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ \ \ /** Limits during query execution are part of the settings. \ @@ -389,7 +390,7 @@ class IColumn; M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ M(Bool, allow_experimental_database_materialize_mysql, false, "Allow to create database with Engine=MaterializeMySQL(...).", 0) \ M(Bool, system_events_show_zero_values, false, "Include all metrics, even with zero values", 0) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ diff --git a/src/DataStreams/SquashingTransform.cpp b/src/DataStreams/SquashingTransform.cpp index c57e2351230..1f6ca8a7306 100644 --- a/src/DataStreams/SquashingTransform.cpp +++ b/src/DataStreams/SquashingTransform.cpp @@ -27,7 +27,7 @@ Block SquashingTransform::add(const Block & input_block) /* * To minimize copying, accept two types of argument: const reference for output - * stream, and rvalue reference for input stream, and decide whether to copy + * stream, and rvalue reference for input stream, and decide whether to copy * inside this function. This allows us not to copy Block unless we absolutely * have to. */ diff --git a/src/DataTypes/NumberTraits.h b/src/DataTypes/NumberTraits.h index 603449150db..77bd2101f05 100644 --- a/src/DataTypes/NumberTraits.h +++ b/src/DataTypes/NumberTraits.h @@ -29,7 +29,7 @@ constexpr size_t min(size_t x, size_t y) } /// @note There's no auto scale to larger big integer, only for integral ones. -/// It's cause of (U)Int64 backward compatibilty and very big performance penalties. +/// It's cause of (U)Int64 backward compatibility and very big performance penalties. constexpr size_t nextSize(size_t size) { if (size < 8) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index a9dbae8ec92..4fcd9f12276 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -116,7 +116,7 @@ void DatabaseAtomic::dropTable(const Context &, const String & table_name, bool } tryRemoveSymlink(table_name); /// Remove the inner table (if any) to avoid deadlock - /// (due to attemp to execute DROP from the worker thread) + /// (due to attempt to execute DROP from the worker thread) if (auto * mv = dynamic_cast(table.get())) mv->dropInnerTable(no_delay); /// Notify DatabaseCatalog that table was dropped. It will remove table data in background. @@ -261,21 +261,29 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora { DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); + bool locked_uuid = false; try { std::unique_lock lock{mutex}; if (query.database != database_name) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed to `{}`, cannot create table in `{}`", database_name, query.database); + /// Do some checks before renaming file from .tmp to .sql not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); - renameNoReplace(table_metadata_tmp_path, table_metadata_path); + /// We will get en exception if some table with the same UUID exists (even if it's detached table or table from another database) + DatabaseCatalog::instance().addUUIDMapping(query.uuid); + locked_uuid = true; + /// It throws if `table_metadata_path` already exists (it's possible if table was detached) + renameNoReplace(table_metadata_tmp_path, table_metadata_path); /// Commit point (a sort of) attachTableUnlocked(query.table, table, lock); /// Should never throw table_name_to_path.emplace(query.table, table_data_path); } catch (...) { Poco::File(table_metadata_tmp_path).remove(); + if (locked_uuid) + DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; } tryCreateSymlink(query.table, table_data_path); diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 81414902a33..0119f17f843 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -329,10 +329,4 @@ const StoragePtr & DatabaseLazyIterator::table() const return current_storage; } -void DatabaseLazyIterator::reset() -{ - if (current_storage) - current_storage.reset(); -} - } diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 58e5e465eef..2d091297c91 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -22,6 +22,10 @@ public: String getEngineName() const override { return "Lazy"; } + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + void loadStoredObjects( Context & context, bool has_force_restore_data_flag, bool force_attach) override; @@ -122,7 +126,6 @@ public: bool isValid() const override; const String & name() const override; const StoragePtr & table() const override; - void reset() override; private: const DatabaseLazy & database; diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 5eacb846d52..357acb32371 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -53,6 +53,9 @@ void DatabaseMemory::dropTable( } table->is_dropped = true; create_queries.erase(table_name); + UUID table_uuid = table->getStorageID().uuid; + if (table_uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().removeUUIDMappingFinally(table_uuid); } ASTPtr DatabaseMemory::getCreateDatabaseQuery() const diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp index ed85028d04d..6c5173c986f 100644 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ b/src/Databases/DatabaseWithDictionaries.cpp @@ -223,6 +223,10 @@ void DatabaseWithDictionaries::removeDictionary(const Context &, const String & attachDictionary(dictionary_name, attach_info); throw; } + + UUID dict_uuid = attach_info.create_query->as()->uuid; + if (dict_uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().removeUUIDMappingFinally(dict_uuid); } DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const FilterByNameFunction & filter_by_dictionary_name) diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 9b744259406..fadec5fe7a9 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -44,8 +44,6 @@ public: /// (a database with support for lazy tables loading /// - it maintains a list of tables but tables are loaded lazily). virtual const StoragePtr & table() const = 0; - /// Reset reference counter to the StoragePtr. - virtual void reset() = 0; virtual ~IDatabaseTablesIterator() = default; @@ -95,8 +93,6 @@ public: const String & name() const override { return it->first; } const StoragePtr & table() const override { return it->second; } - - void reset() override { it->second.reset(); } }; /// Copies list of dictionaries and iterates through such snapshot. @@ -151,6 +147,10 @@ public: /// Get name of database engine. virtual String getEngineName() const = 0; + virtual bool canContainMergeTreeTables() const { return true; } + + virtual bool canContainDistributedTables() const { return true; } + /// Load a set of existing tables. /// You can call only once, right after the object is created. virtual void loadStoredObjects(Context & /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} diff --git a/src/Databases/MySQL/ConnectionMySQLSettings.h b/src/Databases/MySQL/ConnectionMySQLSettings.h index 90279f846a4..ce2773307c5 100644 --- a/src/Databases/MySQL/ConnectionMySQLSettings.h +++ b/src/Databases/MySQL/ConnectionMySQLSettings.h @@ -11,7 +11,7 @@ class Context; class ASTStorage; #define LIST_OF_CONNECTION_MYSQL_SETTINGS(M) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ /// Settings that should not change after the creation of a database. #define APPLY_FOR_IMMUTABLE_CONNECTION_MYSQL_SETTINGS(M) \ diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.h b/src/Databases/MySQL/DatabaseConnectionMySQL.h index 7bf5e8c1d88..d8694e71db2 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.h +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.h @@ -42,6 +42,12 @@ public: String getEngineName() const override { return "MySQL"; } + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + + bool shouldBeEmptyOnDetach() const override { return false; } + bool empty() const override; DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override; diff --git a/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h b/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h index 5a0ec242c2f..86a5cbf8206 100644 --- a/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h +++ b/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h @@ -28,11 +28,6 @@ public: return tables.emplace_back(storage); } - void reset() override - { - tables.clear(); - } - UUID uuid() const override { return nested_iterator->uuid(); } DatabaseMaterializeTablesIterator(DatabaseTablesIteratorPtr nested_iterator_, DatabaseMaterializeMySQL * database_) diff --git a/src/Databases/MySQL/MaterializeMySQLSettings.cpp b/src/Databases/MySQL/MaterializeMySQLSettings.cpp index 609ce011f91..a8672bf488e 100644 --- a/src/Databases/MySQL/MaterializeMySQLSettings.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSettings.cpp @@ -8,7 +8,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -25,9 +24,8 @@ void MaterializeMySQLSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for database " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for database " + storage_def.engine->name); + throw; } } else diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 63df025d2b1..43ff42956cd 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -326,7 +326,7 @@ struct DecimalBinaryOperation } private: - /// there's implicit type convertion here + /// there's implicit type conversion here static NativeResultType apply(NativeResultType a, NativeResultType b) { if constexpr (can_overflow && check_overflow) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 10c4a27e509..68d8b41407d 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -577,7 +577,7 @@ private: auto input_value = input_column->getDataAt(r); if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) { - // empty plaintext results in empty ciphertext + tag, means there should be atleast tag_size bytes. + // empty plaintext results in empty ciphertext + tag, means there should be at least tag_size bytes. if (input_value.size < tag_size) throw Exception(fmt::format("Encrypted data is too short: only {} bytes, " "should contain at least {} bytes of a tag.", diff --git a/src/Functions/formatReadableTimeDelta.cpp b/src/Functions/formatReadableTimeDelta.cpp index e55829b1bad..2b574f672d3 100644 --- a/src/Functions/formatReadableTimeDelta.cpp +++ b/src/Functions/formatReadableTimeDelta.cpp @@ -131,7 +131,7 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - /// Virtual call is Ok (neglible comparing to the rest of calculations). + /// Virtual call is Ok (negligible comparing to the rest of calculations). Float64 value = arguments[0].column->getFloat64(i); bool is_negative = value < 0; diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index 11c413757c6..323c9951a96 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -22,7 +22,7 @@ namespace { /// Returns 1 if and Decimal value has more digits then it's Precision allow, 0 otherwise. -/// Precision could be set as second argument or omitted. If ommited function uses Decimal presicion of the first argument. +/// Precision could be set as second argument or omitted. If omitted function uses Decimal precision of the first argument. class FunctionIsDecimalOverflow : public IFunction { public: diff --git a/src/Functions/lgamma.cpp b/src/Functions/lgamma.cpp index 51b3dfd97df..e4da0d8dfbd 100644 --- a/src/Functions/lgamma.cpp +++ b/src/Functions/lgamma.cpp @@ -4,7 +4,6 @@ #if defined(OS_DARWIN) extern "C" { - /// Is defined in libglibc-compatibility.a double lgamma_r(double x, int * signgamp); } #endif diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 7572968c2c6..22c81d1d365 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -744,12 +744,12 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { function_builder = FunctionFactory::instance().get(node.name, data.context); } - catch (DB::Exception & e) + catch (Exception & e) { auto hints = AggregateFunctionFactory::instance().getHints(node.name); if (!hints.empty()) e.addMessage("Or unknown aggregate function " + node.name + ". Maybe you meant: " + toString(hints)); - e.rethrow(); + throw; } Names argument_names; diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index feb2036a0d6..e1a9a820ebb 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -233,8 +233,8 @@ void AsynchronousMetrics::update() for (const auto & db : databases) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 71567a424c5..37eb403ddab 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -59,6 +59,26 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) writeVarUInt(client_version_patch, out); } + + if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) + { + if (opentelemetry_trace_id) + { + // Have OpenTelemetry header. + writeBinary(uint8_t(1), out); + // No point writing these numbers with variable length, because they + // are random and will probably require the full length anyway. + writeBinary(opentelemetry_trace_id, out); + writeBinary(opentelemetry_span_id, out); + writeBinary(opentelemetry_tracestate, out); + writeBinary(opentelemetry_trace_flags, out); + } + else + { + // Don't have OpenTelemetry header. + writeBinary(uint8_t(0), out); + } + } } @@ -112,6 +132,19 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) else client_version_patch = client_tcp_protocol_version; } + + if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) + { + uint8_t have_trace_id = 0; + readBinary(have_trace_id, in); + if (have_trace_id) + { + readBinary(opentelemetry_trace_id, in); + readBinary(opentelemetry_span_id, in); + readBinary(opentelemetry_tracestate, in); + readBinary(opentelemetry_trace_flags, in); + } + } } @@ -122,6 +155,74 @@ void ClientInfo::setInitialQuery() client_name = (DBMS_NAME " ") + client_name; } +bool ClientInfo::parseTraceparentHeader(const std::string & traceparent, + std::string & error) +{ + uint8_t version = -1; + uint64_t trace_id_high = 0; + uint64_t trace_id_low = 0; + uint64_t trace_parent = 0; + uint8_t trace_flags = 0; + + // Version 00, which is the only one we can parse, is fixed width. Use this + // fact for an additional sanity check. + const int expected_length = 2 + 1 + 32 + 1 + 16 + 1 + 2; + if (traceparent.length() != expected_length) + { + error = fmt::format("unexpected length {}, expected {}", + traceparent.length(), expected_length); + return false; + } + + // clang-tidy doesn't like sscanf: + // error: 'sscanf' used to convert a string to an unsigned integer value, + // but function will not report conversion errors; consider using 'strtoul' + // instead [cert-err34-c,-warnings-as-errors] + // There is no other ready solution, and hand-rolling a more complicated + // parser for an HTTP header in C++ sounds like RCE. + // NOLINTNEXTLINE(cert-err34-c) + int result = sscanf(&traceparent[0], + "%2" SCNx8 "-%16" SCNx64 "%16" SCNx64 "-%16" SCNx64 "-%2" SCNx8, + &version, &trace_id_high, &trace_id_low, &trace_parent, &trace_flags); + + if (result == EOF) + { + error = "EOF"; + return false; + } + + // We read uint128 as two uint64, so 5 parts and not 4. + if (result != 5) + { + error = fmt::format("could only read {} parts instead of the expected 5", + result); + return false; + } + + if (version != 0) + { + error = fmt::format("unexpected version {}, expected 00", version); + return false; + } + + opentelemetry_trace_id = static_cast<__uint128_t>(trace_id_high) << 64 + | trace_id_low; + opentelemetry_span_id = trace_parent; + opentelemetry_trace_flags = trace_flags; + return true; +} + + +std::string ClientInfo::composeTraceparentHeader() const +{ + // This span is a parent for its children, so we specify this span_id as a + // parent id. + return fmt::format("00-{:032x}-{:016x}-{:02x}", opentelemetry_trace_id, + opentelemetry_span_id, + // This cast is needed because fmt is being weird and complaining that + // "mixing character types is not allowed". + static_cast(opentelemetry_trace_flags)); +} void ClientInfo::fillOSUserHostNameAndVersionInfo() { diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 704f1913b89..2edf47684d3 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -58,6 +59,17 @@ public: String initial_query_id; Poco::Net::SocketAddress initial_address; + // OpenTelemetry trace information. + __uint128_t opentelemetry_trace_id = 0; + // The span id we get the in the incoming client info becomes our parent span + // id, and the span id we send becomes downstream parent span id. + UInt64 opentelemetry_span_id = 0; + UInt64 opentelemetry_parent_span_id = 0; + // The incoming tracestate header and the trace flags, we just pass them downstream. + // They are described at https://www.w3.org/TR/trace-context/ + String opentelemetry_tracestate; + UInt8 opentelemetry_trace_flags = 0; + /// All below are parameters related to initial query. Interface interface = Interface::TCP; @@ -90,6 +102,16 @@ public: /// Initialize parameters on client initiating query. void setInitialQuery(); + // Parse/compose OpenTelemetry traceparent header. + // Note that these functions use span_id field, not parent_span_id, same as + // in native protocol. The incoming traceparent corresponds to the upstream + // trace span, and the outgoing traceparent corresponds to our current span. + // We use the same ClientInfo structure first for incoming span, and then + // for our span: when we switch, we use old span_id as parent_span_id, and + // generate a new span_id (currently this happens in Context::setQueryId()). + bool parseTraceparentHeader(const std::string & traceparent, std::string & error); + std::string composeTraceparentHeader() const; + private: void fillOSUserHostNameAndVersionInfo(); }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 9d2ea6ded86..b75e9ae9d58 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1100,28 +1100,53 @@ void Context::setCurrentDatabase(const String & name) void Context::setCurrentQueryId(const String & query_id) { - String query_id_to_set = query_id; + /// Generate random UUID, but using lower quality RNG, + /// because Poco::UUIDGenerator::generateRandom method is using /dev/random, that is very expensive. + /// NOTE: Actually we don't need to use UUIDs for query identifiers. + /// We could use any suitable string instead. + union + { + char bytes[16]; + struct + { + UInt64 a; + UInt64 b; + } words; + __uint128_t uuid; + } random; + random.words.a = thread_local_rng(); //-V656 + random.words.b = thread_local_rng(); //-V656 + + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY + && client_info.opentelemetry_trace_id == 0) + { + // If this is an initial query without any parent OpenTelemetry trace, we + // might start the trace ourselves, with some configurable probability. + std::bernoulli_distribution should_start_trace{ + settings.opentelemetry_start_trace_probability}; + + if (should_start_trace(thread_local_rng)) + { + // Use the randomly generated default query id as the new trace id. + client_info.opentelemetry_trace_id = random.uuid; + client_info.opentelemetry_parent_span_id = 0; + client_info.opentelemetry_span_id = thread_local_rng(); + // Mark this trace as sampled in the flags. + client_info.opentelemetry_trace_flags = 1; + } + } + else + { + // The incoming request has an OpenTelemtry trace context. Its span id + // becomes our parent span id. + client_info.opentelemetry_parent_span_id = client_info.opentelemetry_span_id; + client_info.opentelemetry_span_id = thread_local_rng(); + } + + String query_id_to_set = query_id; if (query_id_to_set.empty()) /// If the user did not submit his query_id, then we generate it ourselves. { - /// Generate random UUID, but using lower quality RNG, - /// because Poco::UUIDGenerator::generateRandom method is using /dev/random, that is very expensive. - /// NOTE: Actually we don't need to use UUIDs for query identifiers. - /// We could use any suitable string instead. - - union - { - char bytes[16]; - struct - { - UInt64 a; - UInt64 b; - } words; - } random; - - random.words.a = thread_local_rng(); //-V656 - random.words.b = thread_local_rng(); //-V656 - /// Use protected constructor. struct QueryUUID : Poco::UUID { @@ -1753,6 +1778,17 @@ std::shared_ptr Context::getAsynchronousMetricLog() } +std::shared_ptr Context::getOpenTelemetrySpanLog() +{ + auto lock = getLock(); + + if (!shared->system_logs) + return {}; + + return shared->system_logs->opentelemetry_span_log; +} + + CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const { auto lock = getLock(); @@ -2016,10 +2052,15 @@ void Context::reloadConfig() const void Context::shutdown() { - for (auto & [disk_name, disk] : getDisksMap()) + // Disk selector might not be initialized if there was some error during + // its initialization. Don't try to initialize it again on shutdown. + if (shared->merge_tree_disk_selector) { - LOG_INFO(shared->log, "Shutdown disk {}", disk_name); - disk->shutdown(); + for (auto & [disk_name, disk] : getDisksMap()) + { + LOG_INFO(shared->log, "Shutdown disk {}", disk_name); + disk->shutdown(); + } } shared->shutdown(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 075fc3837ef..fbf578494ed 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -81,6 +81,7 @@ class TextLog; class TraceLog; class MetricLog; class AsynchronousMetricLog; +class OpenTelemetrySpanLog; struct MergeTreeSettings; class StorageS3Settings; class IDatabase; @@ -541,6 +542,7 @@ public: std::shared_ptr getTextLog(); std::shared_ptr getMetricLog(); std::shared_ptr getAsynchronousMetricLog(); + std::shared_ptr getOpenTelemetrySpanLog(); /// Returns an object used to log operations with parts if it possible. /// Provide table name to make required checks. diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 32d0e25bde5..fc7f5c2f765 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -1246,7 +1246,6 @@ public: size_t num_unfinished_hosts = waiting_hosts.size() - num_hosts_finished; size_t num_active_hosts = current_active_hosts.size(); - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Watching task {} is executing longer than distributed_ddl_task_timeout (={}) seconds. " "There are {} unfinished hosts ({} of them are currently active), they are going to execute the query in background", diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index cfe1046aceb..906863f3f44 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -13,9 +13,16 @@ #include #include #include +#include #include + +namespace CurrentMetrics +{ + extern const Metric TablesToDropQueueSize; +} + namespace DB { @@ -155,7 +162,17 @@ void DatabaseCatalog::shutdownImpl() tables_marked_dropped.clear(); std::lock_guard lock(databases_mutex); - assert(std::find_if_not(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { return elem.map.empty(); }) == uuid_map.end()); + assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) + { + /// Ensure that all UUID mappings are emtpy (i.e. all mappings contain nullptr instead of a pointer to storage) + const auto & not_empty_mapping = [] (const auto & mapping) + { + auto & table = mapping.second.second; + return table; + }; + auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); + return it != elem.map.end(); + }) == uuid_map.end()); databases.clear(); db_uuid_map.clear(); view_dependencies.clear(); @@ -411,36 +428,76 @@ DatabasePtr DatabaseCatalog::getSystemDatabase() const return getDatabase(SYSTEM_DATABASE); } -void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) +void DatabaseCatalog::addUUIDMapping(const UUID & uuid) +{ + addUUIDMapping(uuid, nullptr, nullptr); +} + +void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + assert((database && table) || (!database && !table)); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; - auto [_, inserted] = map_part.map.try_emplace(uuid, std::move(database), std::move(table)); + auto [it, inserted] = map_part.map.try_emplace(uuid, database, table); + if (inserted) + return; + + auto & prev_database = it->second.first; + auto & prev_table = it->second.second; + assert((prev_database && prev_table) || (!prev_database && !prev_table)); + + if (!prev_table && table) + { + /// It's empty mapping, it was created to "lock" UUID and prevent collision. Just update it. + prev_database = database; + prev_table = table; + return; + } + + /// We are trying to replace existing mapping (prev_table != nullptr), it's logical error + if (table) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", toString(uuid)); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR - if (!inserted) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " already exists", ErrorCodes::TABLE_ALREADY_EXISTS); + throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " + "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); } void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + if (it == map_part.map.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); + it->second = {}; +} + +void DatabaseCatalog::removeUUIDMappingFinally(const UUID & uuid) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; if (!map_part.map.erase(uuid)) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); } void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + assert(database && table); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; auto it = map_part.map.find(uuid); if (it == map_part.map.end()) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); - it->second = std::make_pair(std::move(database), std::move(table)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); + auto & prev_database = it->second.first; + auto & prev_table = it->second.second; + assert(prev_database && prev_table); + prev_database = std::move(database); + prev_table = std::move(table); } std::unique_ptr DatabaseCatalog::database_catalog; @@ -631,6 +688,8 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() dropped_metadata.emplace(std::move(full_path), std::move(dropped_id)); } + LOG_INFO(log, "Found {} partially dropped tables. Will load them and retry removal.", dropped_metadata.size()); + ThreadPool pool; for (const auto & elem : dropped_metadata) { @@ -695,6 +754,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr LOG_WARNING(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); } + addUUIDMapping(table_id.uuid); drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); } @@ -704,6 +764,8 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr else tables_marked_dropped.push_back({table_id, table, dropped_metadata_path, drop_time}); tables_marked_dropped_ids.insert(table_id.uuid); + CurrentMetrics::add(CurrentMetrics::TablesToDropQueueSize, 1); + /// If list of dropped tables was empty, start a drop task if (drop_task && tables_marked_dropped.size() == 1) (*drop_task)->schedule(); @@ -732,6 +794,10 @@ void DatabaseCatalog::dropTableDataTask() LOG_INFO(log, "Will try drop {}", table.table_id.getNameForLogs()); tables_marked_dropped.erase(it); } + else + { + LOG_TRACE(log, "Not found any suitable tables to drop, still have {} tables in drop queue", tables_marked_dropped.size()); + } need_reschedule = !tables_marked_dropped.empty(); } catch (...) @@ -770,7 +836,7 @@ void DatabaseCatalog::dropTableDataTask() (*drop_task)->scheduleAfter(reschedule_time_ms); } -void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const +void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) { if (table.table) { @@ -789,6 +855,9 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); Poco::File(table.metadata_path).remove(); + + removeUUIDMappingFinally(table.table_id.uuid); + CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1); } String DatabaseCatalog::getPathForUUID(const UUID & uuid) @@ -826,6 +895,8 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) { if (uuid == UUIDHelpers::Nil) return; + + LOG_DEBUG(log, "Waiting for table {} to be finally dropped", toString(uuid)); std::unique_lock lock{tables_marked_dropped_mutex}; wait_table_finally_dropped.wait(lock, [&]() { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index c6f50117564..d26307a3bc3 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -165,12 +165,21 @@ public: void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where); /// If table has UUID, addUUIDMapping(...) must be called when table attached to some database - /// and removeUUIDMapping(...) must be called when it detached. + /// removeUUIDMapping(...) must be called when it detached, + /// and removeUUIDMappingFinally(...) must be called when table is dropped and its data removed from disk. /// Such tables can be accessed by persistent UUID instead of database and table name. - void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + void addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table); void removeUUIDMapping(const UUID & uuid); + void removeUUIDMappingFinally(const UUID & uuid); /// For moving table between databases void updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + /// This method adds empty mapping (with database and storage equal to nullptr). + /// It's required to "lock" some UUIDs and protect us from collision. + /// Collisions of random 122-bit integers are very unlikely to happen, + /// but we allow to explicitly specify UUID in CREATE query (in particular for testing). + /// If some UUID was already added and we are trying to add it again, + /// this method will throw an exception. + void addUUIDMapping(const UUID & uuid); static String getPathForUUID(const UUID & uuid); @@ -222,7 +231,7 @@ private: void loadMarkedAsDroppedTables(); void dropTableDataTask(); - void dropTableFinally(const TableMarkedAsDropped & table) const; + void dropTableFinally(const TableMarkedAsDropped & table); static constexpr size_t reschedule_time_ms = 100; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 05647aef9f6..762ad6ae575 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -22,6 +22,15 @@ # include "config_core.h" #endif +#include + +#if defined(MEMORY_SANITIZER) + #include +#endif + +#if defined(ADDRESS_SANITIZER) + #include +#endif namespace ProfileEvents { @@ -624,6 +633,22 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } catch (Exception & e) { +#if defined(MEMORY_SANITIZER) + const auto & msg = e.message(); + if (__msan_test_shadow(msg.data(), msg.size()) != -1) + { + LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (msan): {}", e.getStackTraceString()); + } +#endif + +#if defined(ADDRESS_SANITIZER) + const auto & msg = e.message(); + if (__asan_region_is_poisoned(const_cast(msg.data()), msg.size())) + { + LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (asan): {}", e.getStackTraceString()); + } +#endif + e.addMessage(fmt::format("while executing '{}'", action.toString())); throw; } diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index dcef36de175..e1713c7cbbb 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int DICTIONARIES_WAS_NOT_LOADED; } @@ -1404,7 +1405,29 @@ void ExternalLoader::checkLoaded(const ExternalLoader::LoadResult & result, if (result.status == ExternalLoader::Status::LOADING) throw Exception(type_name + " '" + result.name + "' is still loading", ErrorCodes::BAD_ARGUMENTS); if (result.exception) - std::rethrow_exception(result.exception); + { + // Exception is shared for multiple threads. + // Don't just rethrow it, because sharing the same exception object + // between multiple threads can lead to weird effects if they decide to + // modify it, for example, by adding some error context. + try + { + std::rethrow_exception(result.exception); + } + catch (const Poco::Exception & e) + { + /// This will create a copy for Poco::Exception and DB::Exception + e.rethrow(); + } + catch (...) + { + throw DB::Exception(ErrorCodes::DICTIONARIES_WAS_NOT_LOADED, + "Failed to load dictionary '{}': {}", + result.name, + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); + } + } if (result.status == ExternalLoader::Status::NOT_EXIST) throw Exception(type_name + " '" + result.name + "' not found", ErrorCodes::BAD_ARGUMENTS); if (result.status == ExternalLoader::Status::NOT_LOADED) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index c4ebe596649..144e045ecee 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -52,13 +52,37 @@ BlockIO InterpreterDropQuery::execute() return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); } else if (!drop.database.empty()) - return executeToDatabase(drop.database, drop.kind, drop.if_exists, drop.no_delay); + return executeToDatabase(drop); else throw Exception("Nothing to drop, both names are empty", ErrorCodes::LOGICAL_ERROR); } +void InterpreterDropQuery::waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait) +{ + if (uuid_to_wait == UUIDHelpers::Nil) + return; + + if (query.kind == ASTDropQuery::Kind::Drop) + DatabaseCatalog::instance().waitTableFinallyDropped(uuid_to_wait); + else if (query.kind == ASTDropQuery::Kind::Detach) + { + if (auto * atomic = typeid_cast(db.get())) + atomic->waitDetachedTableNotInUse(uuid_to_wait); + } +} + BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) +{ + DatabasePtr database; + UUID table_to_wait_on = UUIDHelpers::Nil; + auto res = executeToTableImpl(query, database, table_to_wait_on); + if (query.no_delay) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_to_wait_on); + return res; +} + +BlockIO InterpreterDropQuery::executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait) { /// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard auto table_id = StorageID(query); @@ -125,19 +149,9 @@ BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) database->dropTable(context, table_id.table_name, query.no_delay); } - } - table.reset(); - ddl_guard = {}; - if (query.no_delay) - { - if (query.kind == ASTDropQuery::Kind::Drop) - DatabaseCatalog::instance().waitTableFinallyDropped(table_id.uuid); - else if (query.kind == ASTDropQuery::Kind::Detach) - { - if (auto * atomic = typeid_cast(database.get())) - atomic->waitDetachedTableNotInUse(table_id.uuid); - } + db = database; + uuid_to_wait = table_id.uuid; } return {}; @@ -223,19 +237,48 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, } -BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay) +BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) { + DatabasePtr database; + std::vector tables_to_wait; + BlockIO res; + try + { + res = executeToDatabaseImpl(query, database, tables_to_wait); + } + catch (...) + { + if (query.no_delay) + { + for (const auto & table_uuid : tables_to_wait) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); + } + throw; + } + + if (query.no_delay) + { + for (const auto & table_uuid : tables_to_wait) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); + } + return res; +} + +BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait) +{ + const auto & database_name = query.database; auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); - if (auto database = tryGetDatabase(database_name, if_exists)) + database = tryGetDatabase(database_name, query.if_exists); + if (database) { - if (kind == ASTDropQuery::Kind::Truncate) + if (query.kind == ASTDropQuery::Kind::Truncate) { throw Exception("Unable to truncate database", ErrorCodes::SYNTAX_ERROR); } - else if (kind == ASTDropQuery::Kind::Detach || kind == ASTDropQuery::Kind::Drop) + else if (query.kind == ASTDropQuery::Kind::Detach || query.kind == ASTDropQuery::Kind::Drop) { - bool drop = kind == ASTDropQuery::Kind::Drop; + bool drop = query.kind == ASTDropQuery::Kind::Drop; context.checkAccess(AccessType::DROP_DATABASE, database_name); if (database->shouldBeEmptyOnDetach()) @@ -246,21 +289,22 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS for (auto iterator = database->getDictionariesIterator(); iterator->isValid(); iterator->next()) { String current_dictionary = iterator->name(); - executeToDictionary(database_name, current_dictionary, kind, false, false, false); + executeToDictionary(database_name, current_dictionary, query.kind, false, false, false); } - ASTDropQuery query; - query.kind = kind; - query.if_exists = true; - query.database = database_name; - query.no_delay = no_delay; + ASTDropQuery query_for_table; + query_for_table.kind = query.kind; + query_for_table.if_exists = true; + query_for_table.database = database_name; + query_for_table.no_delay = query.no_delay; for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { - /// Reset reference counter of the StoragePtr to allow synchronous drop. - iterator->reset(); - query.table = iterator->name(); - executeToTable(query); + DatabasePtr db; + UUID table_to_wait = UUIDHelpers::Nil; + query_for_table.table = iterator->name(); + executeToTableImpl(query_for_table, db, table_to_wait); + uuids_to_wait.push_back(table_to_wait); } } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 0e1fd47b079..fe5362985de 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -29,9 +29,13 @@ private: ASTPtr query_ptr; Context & context; - BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay); + BlockIO executeToDatabase(const ASTDropQuery & query); + BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait); BlockIO executeToTable(const ASTDropQuery & query); + BlockIO executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait); + + static void waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait); BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index ef7fd840ac5..cb5db386f5a 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -50,7 +50,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() return rewritten_query.str(); } - /// SHOW CLUSTER/CLUSTERS + /// SHOW CLUSTER/CLUSTERS if (query.clusters) { std::stringstream rewritten_query; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index f0a8ce9064d..74e3fd0bfa5 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -329,7 +330,8 @@ BlockIO InterpreterSystemQuery::execute() [&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(true); }, [&] () { if (auto text_log = context.getTextLog()) text_log->flush(true); }, [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(true); }, - [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); } + [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); }, + [&] () { if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); } ); break; case Type::STOP_LISTEN_QUERIES: diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 70916fe386d..245feae166d 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -56,7 +56,7 @@ static inline String resolveDatabase( } } - /// When USE other_database_name; CREATE TABLE table_name; + /// When USE other_database_name; CREATE TABLE table_name; /// context.getCurrentDatabase() is always return `default database` /// When USE replica_mysql_database; CREATE TABLE table_name; /// context.getCurrentDatabase() is always return replica_clickhouse_database diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp new file mode 100644 index 00000000000..6c22165546d --- /dev/null +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -0,0 +1,47 @@ +#include "OpenTelemetrySpanLog.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +Block OpenTelemetrySpanLogElement::createBlock() +{ + return { + {std::make_shared(), "trace_id"}, + {std::make_shared(), "span_id"}, + {std::make_shared(), "parent_span_id"}, + {std::make_shared(), "operation_name"}, + {std::make_shared(6), "start_time_us"}, + {std::make_shared(6), "finish_time_us"}, + {std::make_shared(), "finish_date"}, + {std::make_shared(std::make_shared()), + "attribute.names"}, + {std::make_shared(std::make_shared()), + "attribute.values"} + }; +} + +void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const +{ + size_t i = 0; + + columns[i++]->insert(UInt128(Int128(trace_id))); + columns[i++]->insert(span_id); + columns[i++]->insert(parent_span_id); + columns[i++]->insert(operation_name); + columns[i++]->insert(start_time_us); + columns[i++]->insert(finish_time_us); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); + columns[i++]->insert(attribute_names); + columns[i++]->insert(attribute_values); +} + +} + diff --git a/src/Interpreters/OpenTelemetrySpanLog.h b/src/Interpreters/OpenTelemetrySpanLog.h new file mode 100644 index 00000000000..fb382ee3177 --- /dev/null +++ b/src/Interpreters/OpenTelemetrySpanLog.h @@ -0,0 +1,39 @@ +#pragma once + +#include + +namespace DB +{ + +struct OpenTelemetrySpan +{ + __uint128_t trace_id; + UInt64 span_id; + UInt64 parent_span_id; + std::string operation_name; + UInt64 start_time_us; + UInt64 finish_time_us; + UInt64 duration_ns; + Array attribute_names; + Array attribute_values; + // I don't understand how Links work, namely, which direction should they + // point to, and how they are related with parent_span_id, so no Links for + // now. +}; + +struct OpenTelemetrySpanLogElement : public OpenTelemetrySpan +{ + static std::string name() { return "OpenTelemetrySpanLog"; } + static Block createBlock(); + void appendToBlock(MutableColumns & columns) const; +}; + +// OpenTelemetry standartizes some Log data as well, so it's not just +// OpenTelemetryLog to avoid confusion. +class OpenTelemetrySpanLog : public SystemLog +{ +public: + using SystemLog::SystemLog; +}; + +} diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index 0faa748303d..f486752e192 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -16,7 +17,7 @@ struct PreparedSetKey /// if left hand sides of the IN operators have different types). static PreparedSetKey forLiteral(const IAST & ast, DataTypes types_) { - /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, + /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, /// just converts LowCardinality to ordinary types. for (auto & type : types_) type = recursiveRemoveLowCardinality(type); diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index a78b756e291..1e396a0fed1 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,9 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi asynchronous_metric_log = createSystemLog( global_context, "system", "asynchronous_metric_log", config, "asynchronous_metric_log"); + opentelemetry_span_log = createSystemLog( + global_context, "system", "opentelemetry_span_log", config, + "opentelemetry_span_log"); if (query_log) logs.emplace_back(query_log.get()); @@ -104,6 +108,8 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi logs.emplace_back(metric_log.get()); if (asynchronous_metric_log) logs.emplace_back(asynchronous_metric_log.get()); + if (opentelemetry_span_log) + logs.emplace_back(opentelemetry_span_log.get()); try { diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 99a85405348..6c56565a152 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -71,6 +71,7 @@ class TraceLog; class CrashLog; class MetricLog; class AsynchronousMetricLog; +class OpenTelemetrySpanLog; class ISystemLog @@ -105,6 +106,8 @@ struct SystemLogs std::shared_ptr metric_log; /// Used to log all metrics. /// Metrics from system.asynchronous_metrics. std::shared_ptr asynchronous_metric_log; + /// OpenTelemetry trace spans. + std::shared_ptr opentelemetry_span_log; std::vector logs; }; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 2166c78ef7c..7f29cfc7e5c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -166,7 +166,7 @@ void ThreadStatus::initPerformanceCounters() memory_tracker.setDescription("(for thread)"); // query_start_time_{microseconds, nanoseconds} are all constructed from the same time point - // to ensure that they are all equal upto the precision of a second. + // to ensure that they are all equal up to the precision of a second. const auto now = std::chrono::system_clock::now(); query_start_time_nanoseconds = time_in_nanoseconds(now); diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 9d52e30cea4..61ca933dd53 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -644,8 +644,13 @@ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & sou optimizeInjectiveFunctionsInsideUniq(query, context); /// Eliminate min/max/any aggregators of functions of GROUP BY keys - if (settings.optimize_aggregators_of_group_by_keys) + if (settings.optimize_aggregators_of_group_by_keys + && !select_query->group_by_with_totals + && !select_query->group_by_with_rollup + && !select_query->group_by_with_cube) + { optimizeAggregateFunctionsOfGroupByKeys(select_query, query); + } /// Remove duplicate items from ORDER BY. optimizeDuplicatesInOrderBy(select_query); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 57c557c5658..a672b58633d 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -139,15 +140,26 @@ static void logQuery(const String & query, const Context & context, bool interna } else { - const auto & current_query_id = context.getClientInfo().current_query_id; - const auto & initial_query_id = context.getClientInfo().initial_query_id; - const auto & current_user = context.getClientInfo().current_user; + const auto & client_info = context.getClientInfo(); + + const auto & current_query_id = client_info.current_query_id; + const auto & initial_query_id = client_info.initial_query_id; + const auto & current_user = client_info.current_user; LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}) {}", - context.getClientInfo().current_address.toString(), - (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), + client_info.current_address.toString(), + (current_user != "default" ? ", user: " + current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), joinLines(query)); + + if (client_info.opentelemetry_trace_id) + { + LOG_TRACE(&Poco::Logger::get("executeQuery"), + "OpenTelemetry trace id {:x}, span id {}, parent span id {}", + client_info.opentelemetry_trace_id, + client_info.opentelemetry_span_id, + client_info.opentelemetry_parent_span_id); + } } } @@ -194,7 +206,7 @@ inline UInt64 time_in_seconds(std::chrono::time_point return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); } -static void onExceptionBeforeStart(const String & query_for_logging, Context & context, time_t current_time, UInt64 current_time_microseconds, ASTPtr ast) +static void onExceptionBeforeStart(const String & query_for_logging, Context & context, UInt64 current_time_us, ASTPtr ast) { /// Exception before the query execution. if (auto quota = context.getQuota()) @@ -209,11 +221,11 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c // all callers to onExceptionBeforeStart method construct the timespec for event_time and // event_time_microseconds from the same time point. So, it can be assumed that both of these - // times are equal upto the precision of a second. - elem.event_time = current_time; - elem.event_time_microseconds = current_time_microseconds; - elem.query_start_time = current_time; - elem.query_start_time_microseconds = current_time_microseconds; + // times are equal up to the precision of a second. + elem.event_time = current_time_us / 1000000; + elem.event_time_microseconds = current_time_us; + elem.query_start_time = current_time_us / 1000000; + elem.query_start_time_microseconds = current_time_us; elem.current_database = context.getCurrentDatabase(); elem.query = query_for_logging; @@ -233,6 +245,39 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c if (auto query_log = context.getQueryLog()) query_log->add(elem); + if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog(); + context.getClientInfo().opentelemetry_trace_id + && opentelemetry_span_log) + { + OpenTelemetrySpanLogElement span; + span.trace_id = context.getClientInfo().opentelemetry_trace_id; + span.span_id = context.getClientInfo().opentelemetry_span_id; + span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; + span.operation_name = "query"; + span.start_time_us = current_time_us; + span.finish_time_us = current_time_us; + span.duration_ns = 0; + + // keep values synchonized to type enum in QueryLogElement::createBlock + span.attribute_names.push_back("clickhouse.query_status"); + span.attribute_values.push_back("ExceptionBeforeStart"); + + span.attribute_names.push_back("db.statement"); + span.attribute_values.push_back(elem.query); + + span.attribute_names.push_back("clickhouse.query_id"); + span.attribute_values.push_back(elem.client_info.current_query_id); + + if (!context.getClientInfo().opentelemetry_tracestate.empty()) + { + span.attribute_names.push_back("clickhouse.tracestate"); + span.attribute_values.push_back( + context.getClientInfo().opentelemetry_tracestate); + } + + opentelemetry_span_log->add(span); + } + ProfileEvents::increment(ProfileEvents::FailedQuery); if (ast) @@ -266,12 +311,7 @@ static std::tuple executeQueryImpl( bool has_query_tail, ReadBuffer * istr) { - // current_time and current_time_microseconds are both constructed from the same time point - // to ensure that both the times are equal upto the precision of a second. - const auto now = std::chrono::system_clock::now(); - - auto current_time = time_in_seconds(now); - auto current_time_microseconds = time_in_microseconds(now); + const auto current_time = std::chrono::system_clock::now(); /// If we already executing query and it requires to execute internal query, than /// don't replace thread context with given (it can be temporary). Otherwise, attach context to thread. @@ -322,7 +362,7 @@ static std::tuple executeQueryImpl( if (!internal) { - onExceptionBeforeStart(query_for_logging, context, current_time, current_time_microseconds, ast); + onExceptionBeforeStart(query_for_logging, context, time_in_microseconds(current_time), ast); } throw; @@ -494,10 +534,10 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::QUERY_START; - elem.event_time = current_time; - elem.event_time_microseconds = current_time_microseconds; - elem.query_start_time = current_time; - elem.query_start_time_microseconds = current_time_microseconds; + elem.event_time = time_in_seconds(current_time); + elem.event_time_microseconds = time_in_microseconds(current_time); + elem.query_start_time = time_in_seconds(current_time); + elem.query_start_time_microseconds = time_in_microseconds(current_time); elem.current_database = context.getCurrentDatabase(); elem.query = query_for_logging; @@ -568,9 +608,9 @@ static std::tuple executeQueryImpl( // construct event_time and event_time_microseconds using the same time point // so that the two times will always be equal up to a precision of a second. - const auto time_now = std::chrono::system_clock::now(); - elem.event_time = time_in_seconds(time_now); - elem.event_time_microseconds = time_in_microseconds(time_now); + const auto finish_time = std::chrono::system_clock::now(); + elem.event_time = time_in_seconds(finish_time); + elem.event_time_microseconds = time_in_microseconds(finish_time); status_info_to_query_log(elem, info, ast); auto progress_callback = context.getProgressCallback(); @@ -620,6 +660,38 @@ static std::tuple executeQueryImpl( if (auto query_log = context.getQueryLog()) query_log->add(elem); } + + if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog(); + context.getClientInfo().opentelemetry_trace_id + && opentelemetry_span_log) + { + OpenTelemetrySpanLogElement span; + span.trace_id = context.getClientInfo().opentelemetry_trace_id; + span.span_id = context.getClientInfo().opentelemetry_span_id; + span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; + span.operation_name = "query"; + span.start_time_us = elem.query_start_time_microseconds; + span.finish_time_us = time_in_microseconds(finish_time); + span.duration_ns = elapsed_seconds * 1000000000; + + // keep values synchonized to type enum in QueryLogElement::createBlock + span.attribute_names.push_back("clickhouse.query_status"); + span.attribute_values.push_back("QueryFinish"); + + span.attribute_names.push_back("db.statement"); + span.attribute_values.push_back(elem.query); + + span.attribute_names.push_back("clickhouse.query_id"); + span.attribute_values.push_back(elem.client_info.current_query_id); + if (!context.getClientInfo().opentelemetry_tracestate.empty()) + { + span.attribute_names.push_back("clickhouse.tracestate"); + span.attribute_values.push_back( + context.getClientInfo().opentelemetry_tracestate); + } + + opentelemetry_span_log->add(span); + } }; auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota), @@ -631,7 +703,7 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::EXCEPTION_WHILE_PROCESSING; // event_time and event_time_microseconds are being constructed from the same time point - // to ensure that both the times will be equal upto the precision of a second. + // to ensure that both the times will be equal up to the precision of a second. const auto time_now = std::chrono::system_clock::now(); elem.event_time = time_in_seconds(time_now); @@ -694,7 +766,7 @@ static std::tuple executeQueryImpl( if (query_for_logging.empty()) query_for_logging = prepareQueryForLogging(query, context); - onExceptionBeforeStart(query_for_logging, context, current_time, current_time_microseconds, ast); + onExceptionBeforeStart(query_for_logging, context, time_in_microseconds(current_time), ast); } throw; diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 4c0b64934c7..563b7374fdc 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -119,6 +119,7 @@ SRCS( MutationsInterpreter.cpp MySQL/InterpretersMySQLDDLQuery.cpp NullableUtils.cpp + OpenTelemetrySpanLog.cpp OptimizeIfChains.cpp OptimizeIfWithConstantConditionVisitor.cpp PartLog.cpp diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 32382b70bd7..11f4a77ee0e 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -927,7 +927,7 @@ void obfuscateQueries( } else { - /// Everyting else is kept as is. + /// Everything else is kept as is. result.write(token.begin, token.size()); } } diff --git a/src/Processors/IAccumulatingTransform.h b/src/Processors/IAccumulatingTransform.h index 3e77c798ad7..b51753199c3 100644 --- a/src/Processors/IAccumulatingTransform.h +++ b/src/Processors/IAccumulatingTransform.h @@ -36,7 +36,7 @@ public: Status prepare() override; void work() override; - /// Adds additional port fo totals. + /// Adds additional port for totals. /// If added, totals will have been ready by the first generate() call (in totals chunk). InputPort * addTotalsPort(); diff --git a/src/Processors/Transforms/ConvertingTransform.h b/src/Processors/Transforms/ConvertingTransform.h index b426a2ab525..4ae74457998 100644 --- a/src/Processors/Transforms/ConvertingTransform.h +++ b/src/Processors/Transforms/ConvertingTransform.h @@ -1,7 +1,9 @@ #pragma once + #include #include + namespace DB { @@ -46,7 +48,7 @@ private: /// How to construct result block. Position in source block, where to get each column. ColumnNumbers conversion; /// Do not check that constants are same. Use value from result_header. - /// This is needed in case run functions which are constant in query scope, + /// This is needed in case run functions which are constant in query scope, /// but may return different result being executed remotely, like `now64()` or `randConstant()`. /// In this case we replace constants from remote source to constatns from initiator. bool ignore_constant_values; diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 53483d6f7fb..5e03e1d178c 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -95,6 +95,7 @@ namespace ErrorCodes extern const int WRONG_PASSWORD; extern const int REQUIRED_PASSWORD; + extern const int BAD_REQUEST_PARAMETER; extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; } @@ -279,9 +280,7 @@ void HTTPHandler::processQuery( } } - std::string query_id = params.get("query_id", ""); context.setUser(user, password, request.clientAddress()); - context.setCurrentQueryId(query_id); if (!quota_key.empty()) context.setQuotaKey(quota_key); @@ -311,6 +310,31 @@ void HTTPHandler::processQuery( session->release(); }); + // Parse the OpenTelemetry traceparent header. + // Disable in Arcadia -- it interferes with the + // test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test. +#if !defined(ARCADIA_BUILD) + if (request.has("traceparent")) + { + std::string opentelemetry_traceparent = request.get("traceparent"); + std::string error; + if (!context.getClientInfo().parseTraceparentHeader( + opentelemetry_traceparent, error)) + { + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, + "Failed to parse OpenTelemetry traceparent header '{}': {}", + opentelemetry_traceparent, error); + } + + context.getClientInfo().opentelemetry_tracestate = request.get("tracestate", ""); + } +#endif + + // Set the query id supplied by the user, if any, and also update the + // OpenTelemetry fields. + context.setCurrentQueryId(params.get("query_id", + request.get("X-ClickHouse-Query-Id", ""))); + /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); CompressionMethod http_response_compression_method = CompressionMethod::None; diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index bc5436f00ee..1aa5c10afd7 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -43,8 +43,8 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request /// Iterate through all the replicated tables. for (const auto & db : databases) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index a37f88f9306..1b0cbc69b29 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -896,8 +896,6 @@ void TCPHandler::receiveQuery() state.is_empty = false; readStringBinary(state.query_id, *in); - query_context->setCurrentQueryId(state.query_id); - /// Client info ClientInfo & client_info = query_context->getClientInfo(); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) @@ -917,14 +915,6 @@ void TCPHandler::receiveQuery() /// Set fields, that are known apriori. client_info.interface = ClientInfo::Interface::TCP; - if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - { - /// 'Current' fields was set at receiveHello. - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; - } - /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS @@ -1001,11 +991,32 @@ void TCPHandler::receiveQuery() query_context->clampToSettingsConstraints(settings_changes); } query_context->applySettingsChanges(settings_changes); - const Settings & settings = query_context->getSettingsRef(); + + // Use the received query id, or generate a random default. It is convenient + // to also generate the default OpenTelemetry trace id at the same time, and + // set the trace parent. + // Why is this done here and not earlier: + // 1) ClientInfo might contain upstream trace id, so we decide whether to use + // the default ids after we have received the ClientInfo. + // 2) There is the opentelemetry_start_trace_probability setting that + // controls when we start a new trace. It can be changed via Native protocol, + // so we have to apply the changes first. + query_context->setCurrentQueryId(state.query_id); + + // Set parameters of initial query. + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + { + /// 'Current' fields was set at receiveHello. + client_info.initial_user = client_info.current_user; + client_info.initial_query_id = client_info.current_query_id; + client_info.initial_address = client_info.current_address; + } + /// Sync timeouts on client and server during current query to avoid dangling queries on server /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), /// because settings.send_timeout is client-side setting which has opposite meaning on the server side. /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) + const Settings & settings = query_context->getSettingsRef(); state.timeout_setter = std::make_unique(socket(), settings.receive_timeout, settings.send_timeout); } diff --git a/src/Storages/CMakeLists.txt b/src/Storages/CMakeLists.txt index ae47fba063a..deb1c9f6716 100644 --- a/src/Storages/CMakeLists.txt +++ b/src/Storages/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(MergeTree) add_subdirectory(System) if(ENABLE_TESTS) diff --git a/src/Storages/JoinSettings.cpp b/src/Storages/JoinSettings.cpp index 15637d67dea..8a2699746da 100644 --- a/src/Storages/JoinSettings.cpp +++ b/src/Storages/JoinSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void JoinSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/Kafka/KafkaSettings.cpp b/src/Storages/Kafka/KafkaSettings.cpp index 4d80419af2d..6ef74511d83 100644 --- a/src/Storages/Kafka/KafkaSettings.cpp +++ b/src/Storages/Kafka/KafkaSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void KafkaSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/MergeTree/CMakeLists.txt b/src/Storages/MergeTree/CMakeLists.txt new file mode 100644 index 00000000000..36cab0b3590 --- /dev/null +++ b/src/Storages/MergeTree/CMakeLists.txt @@ -0,0 +1,3 @@ +if(ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 319b486c2c6..ffc2dd62ce0 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -353,7 +353,7 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const return checksum->second.file_size; } -String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const +String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const { const auto & storage_columns = metadata_snapshot->getColumns().getAllPhysical(); auto alter_conversions = storage.getAlterConversionsForPart(shared_from_this()); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 202d9494247..21932ba445c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -145,7 +145,7 @@ public: /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const; + String getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const; bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 5b044622b36..ba6c2a3d462 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -68,7 +68,7 @@ MergeInfo MergeListElement::getInfo() const res.memory_usage = memory_tracker.get(); res.thread_id = thread_id; res.merge_type = toString(merge_type); - res.merge_algorithm = toString(merge_algorithm); + res.merge_algorithm = toString(merge_algorithm.load(std::memory_order_relaxed)); for (const auto & source_part_name : source_part_names) res.source_part_names.emplace_back(source_part_name); diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index c1166c55703..09c61250afd 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -92,7 +92,8 @@ struct MergeListElement : boost::noncopyable UInt64 thread_id; MergeType merge_type; - MergeAlgorithm merge_algorithm; + /// Detected after merge already started + std::atomic merge_algorithm; MergeListElement(const std::string & database, const std::string & table, const FutureMergedMutatedPart & future_part); diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 739dfedfde4..ad10a437b1e 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -95,7 +95,7 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetada */ if (!have_at_least_one_physical_column) { - const auto minimum_size_column_name = part->getColumnNameWithMinumumCompressedSize(metadata_snapshot); + const auto minimum_size_column_name = part->getColumnNameWithMinimumCompressedSize(metadata_snapshot); columns.push_back(minimum_size_column_name); /// correctly report added column injected_columns.insert(columns.back()); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index b29966751f9..bc44df5c293 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -298,7 +298,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( if (metadata_snapshot->hasAnyTTL() && merge_with_ttl_allowed && !ttl_merges_blocker.isCancelled()) { - /// TTL delete is prefered to recompression + /// TTL delete is preferred to recompression TTLDeleteMergeSelector delete_ttl_selector( next_delete_ttl_merge_times_by_partition, current_time, @@ -710,10 +710,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; size_t sum_compressed_bytes_upper_bound = merge_entry->total_size_bytes_compressed; - MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); - merge_entry->merge_algorithm = merge_alg; + MergeAlgorithm chosen_merge_algorithm = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); + merge_entry->merge_algorithm.store(chosen_merge_algorithm, std::memory_order_relaxed); - LOG_DEBUG(log, "Selected MergeAlgorithm: {}", toString(merge_alg)); + LOG_DEBUG(log, "Selected MergeAlgorithm: {}", toString(chosen_merge_algorithm)); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex /// (which is locked in data.getTotalActiveSizeInBytes()) @@ -728,7 +728,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor std::unique_ptr rows_sources_write_buf; std::optional column_sizes; - if (merge_alg == MergeAlgorithm::Vertical) + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { tmp_disk->createDirectories(new_part_tmp_path); rows_sources_file_path = new_part_tmp_path + "rows_sources"; @@ -818,7 +818,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ProcessorPtr merged_transform; /// If merge is vertical we cannot calculate it - bool blocks_are_granules_size = (merge_alg == MergeAlgorithm::Vertical); + bool blocks_are_granules_size = (chosen_merge_algorithm == MergeAlgorithm::Vertical); UInt64 merge_block_size = data_settings->merge_max_block_size; switch (data.merging_params.mode) @@ -917,7 +917,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { /// The same progress from merge_entry could be used for both algorithms (it should be more accurate) /// But now we are using inaccurate row-based estimation in Horizontal case for backward compatibility - Float64 progress = (merge_alg == MergeAlgorithm::Horizontal) + Float64 progress = (chosen_merge_algorithm == MergeAlgorithm::Horizontal) ? std::min(1., 1. * rows_written / sum_input_rows_upper_bound) : std::min(1., merge_entry->progress.load(std::memory_order_relaxed)); @@ -938,7 +938,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::DataPart::Checksums checksums_gathered_columns; /// Gather ordinary columns - if (merge_alg == MergeAlgorithm::Vertical) + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { size_t sum_input_rows_exact = merge_entry->rows_read; merge_entry->columns_written = merging_column_names.size(); @@ -1054,7 +1054,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ReadableSize(merge_entry->bytes_read_uncompressed / elapsed_seconds)); } - if (merge_alg != MergeAlgorithm::Vertical) + if (chosen_merge_algorithm != MergeAlgorithm::Vertical) to.writeSuffixAndFinalizePart(new_data_part, need_sync); else to.writeSuffixAndFinalizePart(new_data_part, need_sync, &storage_columns, &checksums_gathered_columns); diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h index f0837f98486..17239e2618a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h @@ -49,7 +49,7 @@ struct MergeTreeDataPartTTLInfos TTLInfoMap recompression_ttl; - /// Return smalles max recompression TTL value + /// Return the smallest max recompression TTL value time_t getMinimalMaxRecompressionTTL() const; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8c1dc845d26..44d0788901f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1517,7 +1517,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { /// In case when SELECT's predicate defines a single continuous interval of keys, /// we can use binary search algorithm to find the left and right endpoint key marks of such interval. - /// The returned value is the minumum range of marks, containing all keys for which KeyCondition holds + /// The returned value is the minimum range of marks, containing all keys for which KeyCondition holds LOG_TRACE(log, "Running binary search on index range for part {} ({} marks)", part->name, marks_count); diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 00580c8d8bb..15ff62e0aa6 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -11,9 +11,8 @@ namespace DB namespace ErrorCodes { - extern const int INVALID_CONFIG_PARAMETER; - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; + extern const int BAD_ARGUMENTS; } IMPLEMENT_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS) @@ -34,9 +33,8 @@ void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::U catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " in MergeTree config", ErrorCodes::INVALID_CONFIG_PARAMETER); - else - e.rethrow(); + e.addMessage("in MergeTree config"); + throw; } } @@ -51,9 +49,8 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 1d133f73a7b..4b5ae580257 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -64,7 +64,7 @@ void MergeTreeWriteAheadLog::addPart(const Block & block, const String & part_na min_block_number = std::min(min_block_number, part_info.min_block); max_block_number = std::max(max_block_number, part_info.max_block); - writeIntBinary(static_cast(0), *out); /// version + writeIntBinary(WAL_VERSION, *out); writeIntBinary(static_cast(ActionType::ADD_PART), *out); writeStringBinary(part_name, *out); block_out->write(block); @@ -80,7 +80,7 @@ void MergeTreeWriteAheadLog::dropPart(const String & part_name) { std::unique_lock lock(write_mutex); - writeIntBinary(static_cast(0), *out); + writeIntBinary(WAL_VERSION, *out); writeIntBinary(static_cast(ActionType::DROP_PART), *out); writeStringBinary(part_name, *out); out->next(); @@ -116,9 +116,13 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor try { + ActionMetadata metadata; + readIntBinary(version, *in); - if (version != 0) - throw Exception("Unknown WAL format version: " + toString(version), ErrorCodes::UNKNOWN_FORMAT_VERSION); + if (version > 0) + { + metadata.read(*in); + } readIntBinary(action_type, *in); readStringBinary(part_name, *in); @@ -233,4 +237,29 @@ MergeTreeWriteAheadLog::tryParseMinMaxBlockNumber(const String & filename) return std::make_pair(min_block, max_block); } +void MergeTreeWriteAheadLog::ActionMetadata::read(ReadBuffer & meta_in) +{ + readIntBinary(min_compatible_version, meta_in); + if (min_compatible_version > WAL_VERSION) + throw Exception("WAL metadata version " + toString(min_compatible_version) + + " is not compatible with this ClickHouse version", ErrorCodes::UNKNOWN_FORMAT_VERSION); + + size_t metadata_size; + readVarUInt(metadata_size, meta_in); + + UInt32 metadata_start = meta_in.offset(); + + /// For the future: read metadata here. + + + /// Skip extra fields if any. If min_compatible_version is lower than WAL_VERSION it means + /// that the fields are not critical for the correctness. + meta_in.ignore(metadata_size - (meta_in.offset() - metadata_start)); +} + +void MergeTreeWriteAheadLog::ActionMetadata::write(WriteBuffer & meta_out) const +{ + writeIntBinary(min_compatible_version, meta_out); + writeVarUInt(static_cast(0), meta_out); +} } diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h index 77c7c7e11e7..f4cf8ddc315 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.h +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.h @@ -28,6 +28,19 @@ public: DROP_PART = 1, }; + struct ActionMetadata + { + /// The minimum version of WAL reader that can understand metadata written by current ClickHouse version. + /// This field must be increased when making backwards incompatible changes. + /// + /// The same approach can be used recursively inside metadata. + UInt8 min_compatible_version = 0; + + void write(WriteBuffer & meta_out) const; + void read(ReadBuffer & meta_in); + }; + + constexpr static UInt8 WAL_VERSION = 0; constexpr static auto WAL_FILE_NAME = "wal"; constexpr static auto WAL_FILE_EXTENSION = ".bin"; constexpr static auto DEFAULT_WAL_FILE_NAME = "wal.bin"; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 45e16e81208..880ad4dd0d3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -57,6 +57,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & return virtual_parts.getContainingPart(data_part->info) != data_part->name; } + bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { auto queue_path = replica_path + "/queue"; @@ -68,6 +69,9 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { std::lock_guard pull_logs_lock(pull_logs_to_queue_mutex); + /// Reset batch size on initialization to recover from possible errors of too large batch size. + current_multi_batch_size = 1; + String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); @@ -486,20 +490,21 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper { std::sort(log_entries.begin(), log_entries.end()); - /// ZK contains a limit on the number or total size of operations in a multi-request. - /// If the limit is exceeded, the connection is simply closed. - /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. - /// The average size of the node value in this case is less than 10 kilobytes. - static constexpr auto MAX_MULTI_OPS = 100; - - for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries; entry_idx += MAX_MULTI_OPS) + for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries;) { auto begin = log_entries.begin() + entry_idx; - auto end = entry_idx + MAX_MULTI_OPS >= log_entries.size() + auto end = entry_idx + current_multi_batch_size >= log_entries.size() ? log_entries.end() - : (begin + MAX_MULTI_OPS); + : (begin + current_multi_batch_size); auto last = end - 1; + /// Increment entry_idx before batch size increase (we copied at most current_multi_batch_size entries) + entry_idx += current_multi_batch_size; + + /// Increase the batch size exponentially, so it will saturate to MAX_MULTI_OPS. + if (current_multi_batch_size < MAX_MULTI_OPS) + current_multi_batch_size = std::min(MAX_MULTI_OPS, current_multi_batch_size * 2); + String last_entry = *last; if (!startsWith(last_entry, "log-")) throw Exception("Error in zookeeper data: unexpected node " + last_entry + " in " + zookeeper_path + "/log", diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 88a61f50225..93b79c8336c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -259,6 +259,19 @@ private: ~CurrentlyExecuting(); }; + /// ZK contains a limit on the number or total size of operations in a multi-request. + /// If the limit is exceeded, the connection is simply closed. + /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. + /// The average size of the node value in this case is less than 10 kilobytes. + static constexpr size_t MAX_MULTI_OPS = 100; + + /// Very large queue entries may appear occasionally. + /// We cannot process MAX_MULTI_OPS at once because it will fail. + /// But we have to process more than one entry at once because otherwise lagged replicas keep up slowly. + /// Let's start with one entry per transaction and icrease it exponentially towards MAX_MULTI_OPS. + /// It will allow to make some progress before failing and remain operational even in extreme cases. + size_t current_multi_batch_size = 1; + public: ReplicatedMergeTreeQueue(StorageReplicatedMergeTree & storage_); ~ReplicatedMergeTreeQueue(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h index 986253a2206..824ed73c171 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h @@ -36,7 +36,7 @@ private: Poco::Logger * log; std::atomic need_stop {false}; - // We need it besides `storage.is_readonly`, bacause `shutdown()` may be called many times, that way `storage.is_readonly` will not change. + // We need it besides `storage.is_readonly`, because `shutdown()` may be called many times, that way `storage.is_readonly` will not change. bool incr_readonly = false; /// The random data we wrote into `/replicas/me/is_active`. diff --git a/src/Storages/MergeTree/TTLMergeSelector.h b/src/Storages/MergeTree/TTLMergeSelector.h index c294687cdc5..710b85b3474 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.h +++ b/src/Storages/MergeTree/TTLMergeSelector.h @@ -84,7 +84,7 @@ public: time_t getTTLForPart(const IMergeSelector::Part & part) const override; /// Checks that part's codec is not already equal to required codec - /// according to recompression TTL. It doesn't make sence to assign such + /// according to recompression TTL. It doesn't make sense to assign such /// merge. bool isTTLAlreadySatisfied(const IMergeSelector::Part & part) const override; private: diff --git a/src/Storages/MergeTree/tests/CMakeLists.txt b/src/Storages/MergeTree/tests/CMakeLists.txt new file mode 100644 index 00000000000..777c75f191a --- /dev/null +++ b/src/Storages/MergeTree/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (wal_action_metadata wal_action_metadata.cpp) +target_link_libraries (wal_action_metadata PRIVATE dbms) diff --git a/src/Storages/MergeTree/tests/wal_action_metadata.cpp b/src/Storages/MergeTree/tests/wal_action_metadata.cpp new file mode 100644 index 00000000000..03c38c7a186 --- /dev/null +++ b/src/Storages/MergeTree/tests/wal_action_metadata.cpp @@ -0,0 +1,61 @@ +#include + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNKNOWN_FORMAT_VERSION; +} +} + +int main(int, char **) +{ + try + { + { + std::cout << "test: dummy test" << std::endl; + + DB::MergeTreeWriteAheadLog::ActionMetadata metadata_out; + DB::MemoryWriteBuffer buf{}; + + metadata_out.write(buf); + buf.finalize(); + + metadata_out.read(*buf.tryGetReadBuffer()); + } + + { + std::cout << "test: min compatibility" << std::endl; + + DB::MergeTreeWriteAheadLog::ActionMetadata metadata_out; + metadata_out.min_compatible_version = DB::MergeTreeWriteAheadLog::WAL_VERSION + 1; + DB::MemoryWriteBuffer buf{}; + + metadata_out.write(buf); + buf.finalize(); + + try + { + metadata_out.read(*buf.tryGetReadBuffer()); + } + catch (const DB::Exception & e) + { + if (e.code() != DB::ErrorCodes::UNKNOWN_FORMAT_VERSION) + { + std::cerr << "Expected UNKNOWN_FORMAT_VERSION exception but got: " + << e.what() << ", " << e.displayText() << std::endl; + } + } + } + } + catch (const DB::Exception & e) + { + std::cerr << e.what() << ", " << e.displayText() << std::endl; + return 1; + } + + return 0; +} diff --git a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp index c74081d8802..830c6224b9e 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp @@ -27,7 +27,7 @@ RabbitMQBlockInputStream::RabbitMQBlockInputStream( , non_virtual_header(metadata_snapshot->getSampleBlockNonMaterialized()) , sample_block(non_virtual_header) , virtual_header(metadata_snapshot->getSampleBlockForColumns( - {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id"}, + {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id", "_timestamp"}, storage.getVirtuals(), storage.getStorageID())) { for (const auto & column : virtual_header) @@ -158,6 +158,7 @@ Block RabbitMQBlockInputStream::readImpl() auto delivery_tag = buffer->getDeliveryTag(); auto redelivered = buffer->getRedelivered(); auto message_id = buffer->getMessageID(); + auto timestamp = buffer->getTimestamp(); buffer->updateAckTracker({delivery_tag, channel_id}); @@ -168,6 +169,7 @@ Block RabbitMQBlockInputStream::readImpl() virtual_columns[2]->insert(delivery_tag); virtual_columns[3]->insert(redelivered); virtual_columns[4]->insert(message_id); + virtual_columns[5]->insert(timestamp); } total_rows = total_rows + new_rows; diff --git a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h index f68b79275f6..5f2c2a62018 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h +++ b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h @@ -30,6 +30,7 @@ public: Block readImpl() override; void readSuffixImpl() override; + bool queueEmpty() const { return !buffer || buffer->queueEmpty(); } bool needChannelUpdate(); void updateChannel(); bool sendAck(); diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.cpp b/src/Storages/RabbitMQ/RabbitMQSettings.cpp index f956c520749..93495cdd8ae 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSettings.cpp @@ -9,7 +9,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -26,9 +25,8 @@ void RabbitMQSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 43a9d75d084..404ba27ccde 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -14,47 +14,27 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING; -} - ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, - ChannelPtr setup_channel_, HandlerPtr event_handler_, - const String & exchange_name_, + std::vector & queues_, size_t channel_id_base_, const String & channel_base_, - const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - bool hash_exchange_, - size_t num_queues_, - const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_) : ReadBuffer(nullptr, 0) , consumer_channel(std::move(consumer_channel_)) - , setup_channel(setup_channel_) , event_handler(event_handler_) - , exchange_name(exchange_name_) + , queues(queues_) , channel_base(channel_base_) , channel_id_base(channel_id_base_) - , queue_base(queue_base_) - , hash_exchange(hash_exchange_) - , num_queues(num_queues_) - , deadletter_exchange(deadletter_exchange_) , log(log_) , row_delimiter(row_delimiter_) - , queue_size(queue_size_) , stopped(stopped_) - , received(queue_size * num_queues) + , received(queue_size_) { - for (size_t queue_id = 0; queue_id < num_queues; ++queue_id) - bindQueue(queue_id); - setupChannel(); } @@ -65,67 +45,6 @@ ReadBufferFromRabbitMQConsumer::~ReadBufferFromRabbitMQConsumer() } -void ReadBufferFromRabbitMQConsumer::bindQueue(size_t queue_id) -{ - std::atomic binding_created = false; - - auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) - { - queues.emplace_back(queue_name); - LOG_DEBUG(log, "Queue {} is declared", queue_name); - - if (msgcount) - LOG_INFO(log, "Queue {} is non-empty. Non-consumed messaged will also be delivered", queue_name); - - /* Here we bind either to sharding exchange (consistent-hash) or to bridge exchange (fanout). All bindings to routing keys are - * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for - * fanout exchange it can be arbitrary - */ - setup_channel->bindQueue(exchange_name, queue_name, std::to_string(channel_id_base)) - .onSuccess([&] { binding_created = true; }) - .onError([&](const char * message) - { - throw Exception( - ErrorCodes::CANNOT_CREATE_RABBITMQ_QUEUE_BINDING, - "Failed to create queue binding with queue {} for exchange {}. Reason: {}", std::string(message), - queue_name, exchange_name); - }); - }; - - auto error_callback([&](const char * message) - { - /* This error is most likely a result of an attempt to declare queue with different settings if it was declared before. So for a - * given queue name either deadletter_exchange parameter changed or queue_size changed, i.e. table was declared with different - * max_block_size parameter. Solution: client should specify a different queue_base parameter or manually delete previously - * declared queues via any of the various cli tools. - */ - throw Exception("Failed to declare queue. Probably queue settings are conflicting: max_block_size, deadletter_exchange. Attempt \ - specifying differently those settings or use a different queue_base or manually delete previously declared queues, \ - which were declared with the same names. ERROR reason: " - + std::string(message), ErrorCodes::BAD_ARGUMENTS); - }); - - AMQP::Table queue_settings; - - queue_settings["x-max-length"] = queue_size; - queue_settings["x-overflow"] = "reject-publish"; - - if (!deadletter_exchange.empty()) - queue_settings["x-dead-letter-exchange"] = deadletter_exchange; - - /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one - * specific queue when its name is specified in queue_base setting - */ - const String queue_name = !hash_exchange ? queue_base : std::to_string(channel_id_base) + "_" + std::to_string(queue_id) + "_" + queue_base; - setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); - - while (!binding_created) - { - iterateEventLoop(); - } -} - - void ReadBufferFromRabbitMQConsumer::subscribe() { for (const auto & queue_name : queues) @@ -146,16 +65,15 @@ void ReadBufferFromRabbitMQConsumer::subscribe() if (row_delimiter != '\0') message_received += row_delimiter; - if (message.hasMessageID()) - received.push({message_received, message.messageID(), redelivered, AckTracker(delivery_tag, channel_id)}); - else - received.push({message_received, "", redelivered, AckTracker(delivery_tag, channel_id)}); + received.push({message_received, message.hasMessageID() ? message.messageID() : "", + message.hasTimestamp() ? message.timestamp() : 0, + redelivered, AckTracker(delivery_tag, channel_id)}); } }) .onError([&](const char * message) { /* End up here either if channel ends up in an error state (then there will be resubscription) or consume call error, which - * arises from queue settings mismatch or queue level error, which should not happen as noone else is supposed to touch them + * arises from queue settings mismatch or queue level error, which should not happen as no one else is supposed to touch them */ LOG_ERROR(log, "Consumer failed on channel {}. Reason: {}", channel_id, message); wait_subscription.store(false); @@ -173,16 +91,16 @@ bool ReadBufferFromRabbitMQConsumer::ackMessages() */ if (record_info.channel_id == channel_id && record_info.delivery_tag && record_info.delivery_tag > prev_tag) { - /// Commit all received messages with delivery tags from last commited to last inserted + /// Commit all received messages with delivery tags from last committed to last inserted if (!consumer_channel->ack(record_info.delivery_tag, AMQP::multiple)) { - LOG_ERROR(log, "Failed to commit messages with delivery tags from last commited to {} on channel {}", + LOG_ERROR(log, "Failed to commit messages with delivery tags from last committed to {} on channel {}", record_info.delivery_tag, channel_id); return false; } prev_tag = record_info.delivery_tag; - LOG_TRACE(log, "Consumer commited messages with deliveryTags up to {} on channel {}", record_info.delivery_tag, channel_id); + LOG_TRACE(log, "Consumer committed messages with deliveryTags up to {} on channel {}", record_info.delivery_tag, channel_id); } return true; diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 109770c77e9..476db3f5e94 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -24,17 +24,12 @@ class ReadBufferFromRabbitMQConsumer : public ReadBuffer public: ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, - ChannelPtr setup_channel_, HandlerPtr event_handler_, - const String & exchange_name_, + std::vector & queues_, size_t channel_id_base_, const String & channel_base_, - const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - bool hash_exchange_, - size_t num_queues_, - const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_); @@ -53,6 +48,7 @@ public: { String message; String message_id; + uint64_t timestamp; bool redelivered; AckTracker track; }; @@ -75,34 +71,26 @@ public: auto getDeliveryTag() const { return current.track.delivery_tag; } auto getRedelivered() const { return current.redelivered; } auto getMessageID() const { return current.message_id; } + auto getTimestamp() const { return current.timestamp; } private: bool nextImpl() override; - void bindQueue(size_t queue_id); void subscribe(); void iterateEventLoop(); ChannelPtr consumer_channel; - ChannelPtr setup_channel; HandlerPtr event_handler; - - const String exchange_name; + std::vector queues; const String channel_base; const size_t channel_id_base; - const String queue_base; - const bool hash_exchange; - const size_t num_queues; - const String deadletter_exchange; Poco::Logger * log; char row_delimiter; bool allowed = true; - uint32_t queue_size; const std::atomic & stopped; String channel_id; std::atomic channel_error = true, wait_subscription = false; - std::vector queues; ConcurrentBoundedQueue received; MessageData current; size_t subscribed = 0; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 6565afeb32d..9735c4d7fd3 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -38,8 +38,10 @@ namespace DB static const auto CONNECT_SLEEP = 200; static const auto RETRIES_MAX = 20; -static const auto HEARTBEAT_RESCHEDULE_MS = 3000; static const uint32_t QUEUE_SIZE = 100000; +static const auto MAX_FAILED_READ_ATTEMPTS = 10; +static const auto RESCHEDULE_MS = 500; +static const auto MAX_THREAD_WORK_DURATION_MS = 60000; namespace ErrorCodes { @@ -50,6 +52,7 @@ namespace ErrorCodes extern const int CANNOT_BIND_RABBITMQ_EXCHANGE; extern const int CANNOT_DECLARE_RABBITMQ_EXCHANGE; extern const int CANNOT_REMOVE_RABBITMQ_EXCHANGE; + extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING; } namespace ExchangeType @@ -122,9 +125,6 @@ StorageRabbitMQ::StorageRabbitMQ( streaming_task = global_context.getSchedulePool().createTask("RabbitMQStreamingTask", [this]{ streamingToViewsFunc(); }); streaming_task->deactivate(); - heartbeat_task = global_context.getSchedulePool().createTask("RabbitMQHeartbeatTask", [this]{ heartbeatFunc(); }); - heartbeat_task->deactivate(); - if (queue_base.empty()) { /* Make sure that local exchange name is unique for each table and is not the same as client's exchange name. It also needs to @@ -210,16 +210,6 @@ Context StorageRabbitMQ::addSettings(Context context) const } -void StorageRabbitMQ::heartbeatFunc() -{ - if (!stream_cancelled && event_handler->connectionRunning()) - { - connection->heartbeat(); - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); - } -} - - void StorageRabbitMQ::loopingFunc() { if (event_handler->connectionRunning()) @@ -396,17 +386,77 @@ void StorageRabbitMQ::bindExchange() } +void StorageRabbitMQ::bindQueue(size_t queue_id) +{ + std::atomic binding_created = false; + + auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) + { + queues.emplace_back(queue_name); + LOG_DEBUG(log, "Queue {} is declared", queue_name); + + if (msgcount) + LOG_INFO(log, "Queue {} is non-empty. Non-consumed messaged will also be delivered", queue_name); + + /* Here we bind either to sharding exchange (consistent-hash) or to bridge exchange (fanout). All bindings to routing keys are + * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for + * fanout exchange it can be arbitrary + */ + setup_channel->bindQueue(consumer_exchange, queue_name, std::to_string(queue_id)) + .onSuccess([&] { binding_created = true; }) + .onError([&](const char * message) + { + throw Exception( + ErrorCodes::CANNOT_CREATE_RABBITMQ_QUEUE_BINDING, + "Failed to create queue binding for exchange {}. Reason: {}", exchange_name, std::string(message)); + }); + }; + + auto error_callback([&](const char * message) + { + /* This error is most likely a result of an attempt to declare queue with different settings if it was declared before. So for a + * given queue name either deadletter_exchange parameter changed or queue_size changed, i.e. table was declared with different + * max_block_size parameter. Solution: client should specify a different queue_base parameter or manually delete previously + * declared queues via any of the various cli tools. + */ + throw Exception("Failed to declare queue. Probably queue settings are conflicting: max_block_size, deadletter_exchange. Attempt \ + specifying differently those settings or use a different queue_base or manually delete previously declared queues, \ + which were declared with the same names. ERROR reason: " + + std::string(message), ErrorCodes::BAD_ARGUMENTS); + }); + + AMQP::Table queue_settings; + + queue_settings["x-max-length"] = queue_size; + + if (!deadletter_exchange.empty()) + queue_settings["x-dead-letter-exchange"] = deadletter_exchange; + else + queue_settings["x-overflow"] = "reject-publish"; + + /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one + * specific queue when its name is specified in queue_base setting + */ + const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; + setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); + + while (!binding_created) + { + event_handler->iterateLoop(); + } +} + + bool StorageRabbitMQ::restoreConnection(bool reconnecting) { size_t cnt_retries = 0; if (reconnecting) { - deactivateTask(heartbeat_task, false, false); connection->close(); /// Connection might be unusable, but not closed /* Connection is not closed immediately (firstly, all pending operations are completed, and then - * an AMQP closing-handshake is performed). But cannot open a new connection untill previous one is properly closed + * an AMQP closing-handshake is performed). But cannot open a new connection until previous one is properly closed */ while (!connection->closed() && ++cnt_retries != RETRIES_MAX) event_handler->iterateLoop(); @@ -452,11 +502,11 @@ void StorageRabbitMQ::unbindExchange() */ std::call_once(flag, [&]() { - heartbeat_task->deactivate(); streaming_task->deactivate(); event_handler->updateLoopState(Loop::STOP); looping_task->deactivate(); + setup_channel = std::make_shared(connection.get()); setup_channel->removeExchange(bridge_exchange) .onSuccess([&]() { @@ -471,6 +521,8 @@ void StorageRabbitMQ::unbindExchange() { event_handler->iterateLoop(); } + + setup_channel->close(); }); } @@ -499,8 +551,6 @@ Pipe StorageRabbitMQ::read( deactivateTask(looping_task, false, true); update_channels = restoreConnection(true); - if (update_channels) - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); } Pipes pipes; @@ -521,7 +571,6 @@ Pipe StorageRabbitMQ::read( if (event_handler->loopRunning()) { deactivateTask(looping_task, false, true); - deactivateTask(heartbeat_task, false, false); } rabbit_stream->updateChannel(); @@ -552,6 +601,13 @@ void StorageRabbitMQ::startup() initExchange(); bindExchange(); + for (size_t i = 1; i <= num_queues; ++i) + { + bindQueue(i); + } + + setup_channel->close(); + for (size_t i = 0; i < num_consumers; ++i) { try @@ -568,7 +624,6 @@ void StorageRabbitMQ::startup() event_handler->updateLoopState(Loop::RUN); streaming_task->activateAndSchedule(); - heartbeat_task->activateAndSchedule(); } @@ -579,7 +634,6 @@ void StorageRabbitMQ::shutdown() deactivateTask(streaming_task, true, false); deactivateTask(looping_task, true, true); - deactivateTask(heartbeat_task, true, false); connection->close(); @@ -635,9 +689,8 @@ ConsumerBufferPtr StorageRabbitMQ::createReadBuffer() ChannelPtr consumer_channel = std::make_shared(connection.get()); return std::make_shared( - consumer_channel, setup_channel, event_handler, consumer_exchange, ++consumer_id, - unique_strbase, queue_base, log, row_delimiter, hash_exchange, num_queues, - deadletter_exchange, queue_size, stream_cancelled); + consumer_channel, event_handler, queues, ++consumer_id, + unique_strbase, log, row_delimiter, queue_size, stream_cancelled); } @@ -683,11 +736,14 @@ void StorageRabbitMQ::streamingToViewsFunc() try { auto table_id = getStorageID(); + // Check if at least one direct dependency is attached size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); if (dependencies_count) { + auto start_time = std::chrono::steady_clock::now(); + // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0) { @@ -696,8 +752,17 @@ void StorageRabbitMQ::streamingToViewsFunc() LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); - if (!streamToViews()) + if (streamToViews()) break; + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time); + if (duration.count() > MAX_THREAD_WORK_DURATION_MS) + { + event_handler->updateLoopState(Loop::STOP); + LOG_TRACE(log, "Reschedule streaming. Thread work duration limit exceeded."); + break; + } } } } @@ -708,7 +773,7 @@ void StorageRabbitMQ::streamingToViewsFunc() /// Wait for attached views if (!stream_cancelled) - streaming_task->schedule(); + streaming_task->scheduleAfter(RESCHEDULE_MS); } @@ -731,13 +796,6 @@ bool StorageRabbitMQ::streamToViews() auto column_names = block_io.out->getHeader().getNames(); auto sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); - /* event_handler->connectionRunning() does not guarantee that connnection is not closed in case loop was not running before, but - * need to anyway start the loop to activate error callbacks and update connection state, because even checking with - * connection->usable() will not give correct answer before callbacks are activated. - */ - if (!event_handler->loopRunning() && event_handler->connectionRunning()) - looping_task->activateAndSchedule(); - auto block_size = getMaxBlockSize(); // Create a stream for each consumer and join them in a union stream @@ -770,34 +828,45 @@ bool StorageRabbitMQ::streamToViews() in = streams[0]; std::atomic stub = {false}; + + if (!event_handler->loopRunning()) + { + event_handler->updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); + } + copyData(*in, *block_io.out, &stub); - /* Need to stop loop even if connection is ok, because sending ack() with loop running in another thread will lead to a lot of data - * races inside the library, but only in case any error occurs or connection is lost while ack is being sent + /* Note: sending ack() with loop running in another thread will lead to a lot of data races inside the library, but only in case + * error occurs or connection is lost while ack is being sent */ - if (event_handler->loopRunning()) - deactivateTask(looping_task, false, true); + deactivateTask(looping_task, false, true); + size_t queue_empty = 0; if (!event_handler->connectionRunning()) { - if (!stream_cancelled && restoreConnection(true)) + if (stream_cancelled) + return true; + + if (restoreConnection(true)) { for (auto & stream : streams) stream->as()->updateChannel(); } else { - /// Reschedule if unable to connect to rabbitmq or quit if cancelled - return false; + LOG_TRACE(log, "Reschedule streaming. Unable to restore connection."); + return true; } } else { - deactivateTask(heartbeat_task, false, false); - /// Commit for (auto & stream : streams) { + if (stream->as()->queueEmpty()) + ++queue_empty; + /* false is returned by the sendAck function in only two cases: * 1) if connection failed. In this case all channels will be closed and will be unable to send ack. Also ack is made based on * delivery tags, which are unique to channels, so if channels fail, those delivery tags will become invalid and there is @@ -828,19 +897,25 @@ bool StorageRabbitMQ::streamToViews() break; } } + + event_handler->iterateLoop(); } } - event_handler->updateLoopState(Loop::RUN); - looping_task->activateAndSchedule(); - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); /// It is also deactivated in restoreConnection(), so reschedule anyway + if ((queue_empty == num_created_consumers) && (++read_attempts == MAX_FAILED_READ_ATTEMPTS)) + { + connection->heartbeat(); + read_attempts = 0; + LOG_TRACE(log, "Reschedule streaming. Queues are empty."); + return true; + } + else + { + event_handler->updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); + } - // Check whether the limits were applied during query execution - bool limits_applied = false; - const BlockStreamProfileInfo & info = in->getProfileInfo(); - limits_applied = info.hasAppliedLimit(); - - return limits_applied; + return false; } @@ -907,7 +982,8 @@ NamesAndTypesList StorageRabbitMQ::getVirtuals() const {"_channel_id", std::make_shared()}, {"_delivery_tag", std::make_shared()}, {"_redelivered", std::make_shared()}, - {"_message_id", std::make_shared()} + {"_message_id", std::make_shared()}, + {"_timestamp", std::make_shared()} }; } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 8d9a20f9e34..d7891aed0a7 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -114,14 +114,15 @@ private: std::atomic wait_confirm = true; /// needed to break waiting for confirmations for producer std::atomic exchange_removed = false; ChannelPtr setup_channel; + std::vector queues; std::once_flag flag; /// remove exchange only once std::mutex task_mutex; BackgroundSchedulePool::TaskHolder streaming_task; - BackgroundSchedulePool::TaskHolder heartbeat_task; BackgroundSchedulePool::TaskHolder looping_task; std::atomic stream_cancelled{false}; + size_t read_attempts = 0; ConsumerBufferPtr createReadBuffer(); @@ -140,6 +141,7 @@ private: void initExchange(); void bindExchange(); + void bindQueue(size_t queue_id); bool restoreConnection(bool reconnecting); bool streamToViews(); diff --git a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h index a8e94070dbd..28fa5df8111 100644 --- a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h +++ b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h @@ -76,20 +76,20 @@ private: String channel_id; /* payloads.queue: - * - payloads are pushed to queue in countRow and poped by another thread in writingFunc, each payload gets into queue only once + * - payloads are pushed to queue in countRow and popped by another thread in writingFunc, each payload gets into queue only once * returned.queue: * - payloads are pushed to queue: * 1) inside channel->onError() callback if channel becomes unusable and the record of pending acknowledgements from server * is non-empty. * 2) inside removeRecord() if received nack() - negative acknowledgement from the server that message failed to be written * to disk or it was unable to reach the queue. - * - payloads are poped from the queue once republished + * - payloads are popped from the queue once republished */ ConcurrentBoundedQueue> payloads, returned; /* Counter of current delivery on a current channel. Delivery tags are scoped per channel. The server attaches a delivery tag for each * published message - a serial number of delivery on current channel. Delivery tag is a way of server to notify publisher if it was - * able or unable to process delivery, i.e. it sends back a responce with a corresponding delivery tag. + * able or unable to process delivery, i.e. it sends back a response with a corresponding delivery tag. */ UInt64 delivery_tag = 0; @@ -100,7 +100,7 @@ private: */ bool wait_all = true; - /* false: untill writeSuffix is called + /* false: until writeSuffix is called * true: means payloads.queue will not grow anymore */ std::atomic wait_num = 0; diff --git a/src/Storages/SetSettings.cpp b/src/Storages/SetSettings.cpp index f7ff1c446f2..baa3d231067 100644 --- a/src/Storages/SetSettings.cpp +++ b/src/Storages/SetSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void SetSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9046940b3f7..0c1561fca9b 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -80,7 +80,6 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int TOO_MANY_ROWS; extern const int UNABLE_TO_SKIP_UNUSED_SHARDS; - extern const int LOGICAL_ERROR; } namespace ActionLocks @@ -600,15 +599,22 @@ void StorageDistributed::shutdown() monitors_blocker.cancelForever(); std::lock_guard lock(cluster_nodes_mutex); + + LOG_DEBUG(log, "Joining background threads for async INSERT"); cluster_nodes_data.clear(); + LOG_DEBUG(log, "Background threads for async INSERT joined"); } void StorageDistributed::drop() { - // shutdown() should be already called - // and by the same reason we cannot use truncate() here, since - // cluster_nodes_data already cleaned - if (!cluster_nodes_data.empty()) - throw Exception("drop called before shutdown", ErrorCodes::LOGICAL_ERROR); + // Some INSERT in-between shutdown() and drop() can call + // requireDirectoryMonitor() again, so call shutdown() to clear them, but + // when the drop() (this function) executed none of INSERT is allowed in + // parallel. + // + // And second time shutdown() should be fast, since none of + // DirectoryMonitor should do anything, because ActionBlocker is canceled + // (in shutdown()). + shutdown(); // Distributed table w/o sharding_key does not allows INSERTs if (relative_data_path.empty()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index edc8a9df911..4d65fe61dc1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3481,8 +3481,10 @@ void StorageReplicatedMergeTree::startup() { queue.initialize(getDataParts()); - data_parts_exchange_endpoint = std::make_shared(*this); - global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint); + InterserverIOEndpointPtr data_parts_exchange_ptr = std::make_shared(*this); + [[maybe_unused]] auto prev_ptr = std::atomic_exchange(&data_parts_exchange_endpoint, data_parts_exchange_ptr); + assert(prev_ptr == nullptr); + global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_ptr->getId(replica_path), data_parts_exchange_ptr); /// In this thread replica will be activated. restarting_thread.start(); @@ -3549,15 +3551,15 @@ void StorageReplicatedMergeTree::shutdown() global_context.getBackgroundMovePool().removeTask(move_parts_task_handle); move_parts_task_handle.reset(); - if (data_parts_exchange_endpoint) + auto data_parts_exchange_ptr = std::atomic_exchange(&data_parts_exchange_endpoint, InterserverIOEndpointPtr{}); + if (data_parts_exchange_ptr) { - global_context.getInterserverIOHandler().removeEndpointIfExists(data_parts_exchange_endpoint->getId(replica_path)); + global_context.getInterserverIOHandler().removeEndpointIfExists(data_parts_exchange_ptr->getId(replica_path)); /// Ask all parts exchange handlers to finish asap. New ones will fail to start - data_parts_exchange_endpoint->blocker.cancelForever(); + data_parts_exchange_ptr->blocker.cancelForever(); /// Wait for all of them - std::unique_lock lock(data_parts_exchange_endpoint->rwlock); + std::unique_lock lock(data_parts_exchange_ptr->rwlock); } - data_parts_exchange_endpoint.reset(); /// We clear all old parts after stopping all background operations. It's /// important, because background operations can produce temporary parts @@ -5900,7 +5902,10 @@ ActionLock StorageReplicatedMergeTree::getActionLock(StorageActionBlockType acti return fetcher.blocker.cancel(); if (action_type == ActionLocks::PartsSend) - return data_parts_exchange_endpoint ? data_parts_exchange_endpoint->blocker.cancel() : ActionLock(); + { + auto data_parts_exchange_ptr = std::atomic_load(&data_parts_exchange_endpoint); + return data_parts_exchange_ptr ? data_parts_exchange_ptr->blocker.cancel() : ActionLock(); + } if (action_type == ActionLocks::ReplicationQueue) return queue.actions_blocker.cancel(); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index a6c8f1b39f8..55c16496ba5 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -67,6 +67,22 @@ namespace const CompressionMethod compression_method) : SourceWithProgress(sample_block), name(std::move(name_)) { + ReadWriteBufferFromHTTP::HTTPHeaderEntries header; + + // Propagate OpenTelemetry trace context, if any, downstream. + const auto & client_info = context.getClientInfo(); + if (client_info.opentelemetry_trace_id) + { + header.emplace_back("traceparent", + client_info.composeTraceparentHeader()); + + if (!client_info.opentelemetry_tracestate.empty()) + { + header.emplace_back("tracestate", + client_info.opentelemetry_tracestate); + } + } + read_buf = wrapReadBufferWithCompressionMethod( std::make_unique( uri, @@ -76,7 +92,7 @@ namespace context.getSettingsRef().max_http_get_redirects, Poco::Net::HTTPBasicCredentials{}, DBMS_DEFAULT_BUFFER_SIZE, - ReadWriteBufferFromHTTP::HTTPHeaderEntries{}, + header, context.getRemoteHostFilter()), compression_method); diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 2459be0ba71..39ccea64e26 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -38,8 +38,8 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, cons std::map> tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain distributed tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain distributed tables + if (!db.second->canContainDistributedTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index ffa789a4751..93bc16785b2 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -32,8 +32,8 @@ static StorageSystemGraphite::Configs getConfigs(const Context & context) for (const auto & db : databases) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 32f672b8401..f66f57ef5d1 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -44,8 +44,8 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex std::map> merge_tree_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index faa2ec0e1c3..d10346af89f 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -83,9 +83,9 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const MutableColumnPtr database_column_mut = ColumnString::create(); for (const auto & database : databases) { - /// Lazy database can not contain MergeTree tables - /// and it's unnecessary to load all tables of Lazy database just to filter all of them. - if (database.second->getEngineName() != "Lazy") + /// Checck if database can contain MergeTree tables, + /// if not it's unnecessary to load all tables of database just to filter all of them. + if (database.second->canContainMergeTreeTables()) database_column_mut->insert(database.first); } block_to_filter.insert(ColumnWithTypeAndName( diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 7ab6e939815..ab54d760873 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -74,8 +74,8 @@ Pipe StorageSystemReplicas::read( std::map> replicated_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index f04d8759507..9cd5e8b8ff3 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -55,8 +55,8 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const std::map> replicated_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 418b8ce0356..4e5f36a5f5d 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -333,7 +333,7 @@ "with_coverage": false } }, - "Functional stateless tests (release, polymorphic parts enabled)": { + "Functional stateless tests (release, wide parts enabled)": { "required_build_properties": { "compiler": "clang-11", "package_type": "deb", diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 5d81575ef38..3702268819b 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -181,7 +181,7 @@ def need_retry(stderr): def get_processlist(client_cmd): try: - return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True) + return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True).decode('utf-8') except: return "" # server seems dead @@ -190,7 +190,7 @@ def get_processlist(client_cmd): def get_stacktraces_from_gdb(server_pid): cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) try: - return subprocess.check_output(cmd, shell=True) + return subprocess.check_output(cmd, shell=True).decode('utf-8') except Exception as ex: return "Error occured while receiving stack traces from gdb: {}".format(str(ex)) @@ -199,7 +199,10 @@ def get_stacktraces_from_gdb(server_pid): # it does not work in Sandbox def get_stacktraces_from_clickhouse(client): try: - return subprocess.check_output("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True) + return subprocess.check_output("{} --allow_introspection_functions=1 --query " + "\"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), " + "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " + "FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8') except Exception as ex: return "Error occured while receiving stack traces from client: {}".format(str(ex)) diff --git a/tests/config/install.sh b/tests/config/install.sh index ef9604904e7..ff96e46c947 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -30,6 +30,7 @@ ln -sf $SRC_PATH/config.d/database_atomic.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/ +ln -sf $SRC_PATH/users.d/database_atomic_drop_detach_sync.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/ints_dictionary.xml $DEST_SERVER_PATH/ ln -sf $SRC_PATH/strings_dictionary.xml $DEST_SERVER_PATH/ diff --git a/tests/config/users.d/database_atomic_drop_detach_sync.xml b/tests/config/users.d/database_atomic_drop_detach_sync.xml new file mode 100644 index 00000000000..4313edf8be1 --- /dev/null +++ b/tests/config/users.d/database_atomic_drop_detach_sync.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/README.md b/tests/integration/README.md index bc64b686782..0886dc2cfac 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -14,7 +14,32 @@ Don't use Docker from your system repository. * [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev` * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest` -* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: `sudo -H pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio confluent-kafka avro +* [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: + +``` +sudo -H pip install \ + PyMySQL \ + aerospike \ + avro \ + cassandra-driver \ + confluent-kafka \ + dicttoxml \ + docker \ + docker-compose==1.22.0 \ + grpcio \ + grpcio-tools \ + kafka-python \ + kazoo \ + minio \ + protobuf \ + psycopg2-binary==2.7.5 \ + pymongo \ + pytest \ + pytest-timeout \ + redis \ + tzlocal \ + urllib3 +``` (highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio` diff --git a/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml b/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/tests/integration/test_concurrent_ttl_merges/test.py b/tests/integration/test_concurrent_ttl_merges/test.py index f067e65f58a..65bc3828b38 100644 --- a/tests/integration/test_concurrent_ttl_merges/test.py +++ b/tests/integration/test_concurrent_ttl_merges/test.py @@ -5,8 +5,8 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml', 'configs/log_conf.xml'], with_zookeeper=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_disabled_mysql_server/__init__.py b/tests/integration/test_disabled_mysql_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_disabled_mysql_server/configs/remote_servers.xml b/tests/integration/test_disabled_mysql_server/configs/remote_servers.xml new file mode 100644 index 00000000000..de8e5865f12 --- /dev/null +++ b/tests/integration/test_disabled_mysql_server/configs/remote_servers.xml @@ -0,0 +1,12 @@ + + + + + + node1 + 9000 + + + + + diff --git a/tests/integration/test_disabled_mysql_server/test.py b/tests/integration/test_disabled_mysql_server/test.py new file mode 100644 index 00000000000..a2cbcb17534 --- /dev/null +++ b/tests/integration/test_disabled_mysql_server/test.py @@ -0,0 +1,60 @@ +import time +import contextlib +import pymysql.cursors +import pytest +import os +import subprocess + +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster, get_docker_compose_path +from helpers.network import PartitionManager + +cluster = ClickHouseCluster(__file__) +clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +class MySQLNodeInstance: + def __init__(self, user='root', password='clickhouse', hostname='127.0.0.1', port=3308): + self.user = user + self.port = port + self.hostname = hostname + self.password = password + self.mysql_connection = None # lazy init + + def alloc_connection(self): + if self.mysql_connection is None: + self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, + port=self.port, autocommit=True) + return self.mysql_connection + + def query(self, execution_query): + with self.alloc_connection().cursor() as cursor: + cursor.execute(execution_query) + + def close(self): + if self.mysql_connection is not None: + self.mysql_connection.close() + + +def test_disabled_mysql_server(started_cluster): + with contextlib.closing(MySQLNodeInstance()) as mysql_node: + mysql_node.query("CREATE DATABASE test_db;") + mysql_node.query("CREATE TABLE test_db.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + + with PartitionManager() as pm: + clickhouse_node.query("CREATE DATABASE test_db ENGINE = MySQL('mysql1:3306', 'test_db', 'root', 'clickhouse')") + + pm._add_rule({'source': clickhouse_node.ip_address, 'destination_port': 3306, 'action': 'DROP'}) + clickhouse_node.query("SELECT * FROM system.parts") + clickhouse_node.query("SELECT * FROM system.mutations") + clickhouse_node.query("SELECT * FROM system.graphite_retentions") + + clickhouse_node.query("DROP DATABASE test_db") diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index c748653bc82..ad09519a484 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -21,8 +21,8 @@ def cluster(): def test_different_types(cluster): node = cluster.instances["node"] - responce = node.query("SELECT * FROM system.disks") - disks = responce.split("\n") + response = node.query("SELECT * FROM system.disks") + disks = response.split("\n") for disk in disks: if disk == '': # skip empty line (after split at last position) continue diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py index e7b86a210bd..0758dc38ba7 100644 --- a/tests/integration/test_distributed_load_balancing/test.py +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -26,7 +26,7 @@ def bootstrap(): # just after server starts (+ 2 seconds, reload timeout). # # And on configuration reload the clusters will be re-created, so some - # internal stuff will be reseted: + # internal stuff will be reset: # - error_count # - last_used (round_robing) # diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 496b34f22f0..07478d99657 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -286,7 +286,7 @@ def test_query_parser(start_cluster): node1.query( "ALTER TABLE table_with_normal_policy MODIFY SETTING storage_policy='moving_jbod_with_external'") finally: - node1.query("DROP TABLE IF EXISTS table_with_normal_policy") + node1.query("DROP TABLE IF EXISTS table_with_normal_policy SYNC") @pytest.mark.parametrize("name,engine", [ @@ -327,7 +327,7 @@ def test_alter_policy(start_cluster, name, engine): name=name)) == "jbods_with_external\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def get_random_string(length): @@ -355,9 +355,7 @@ def test_no_warning_about_zero_max_data_part_size(start_cluster): ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' """) - node.query(""" - DROP TABLE default.test_warning_table - """) + node.query("DROP TABLE default.test_warning_table SYNC") log = get_log(node) assert not re.search("Warning.*Volume.*special_warning_zero_volume", log) assert not re.search("Warning.*Volume.*special_warning_default_volume", log) @@ -398,7 +396,7 @@ def test_round_robin(start_cluster, name, engine): assert used_disks[0] != used_disks[1] assert used_disks[2] == used_disks[0] finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -425,7 +423,7 @@ def test_max_data_part_size(start_cluster, name, engine): assert len(used_disks) == 1 assert used_disks[0] == 'external' finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -451,8 +449,11 @@ def test_jbod_overflow(start_cluster, name, engine): data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + for p in ("/jbod1", "/jbod2", "/external"): + print(node1.exec_in_container([f"bash", "-c", f"find {p} | xargs -n1 du -sh"])) + used_disks = get_used_disks_for_table(node1, name) - assert all(disk == 'jbod1' for disk in used_disks) + assert set(used_disks) == {'jbod1'} # should go to the external disk (jbod is overflown) data = [] # 10MB in total @@ -461,6 +462,9 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + for p in ("/jbod1", "/jbod2", "/external"): + print(node1.exec_in_container([f"bash", "-c", f"find {p} | xargs -n1 du -sh"])) + used_disks = get_used_disks_for_table(node1, name) assert used_disks[-1] == 'external' @@ -478,7 +482,7 @@ def test_jbod_overflow(start_cluster, name, engine): assert all(disk == 'external' for disk in disks_for_merges) finally: - node1.query("DROP TABLE IF EXISTS {}".format(name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -495,6 +499,8 @@ def test_background_move(start_cluster, name, engine): SETTINGS storage_policy='moving_jbod_with_external' """.format(name=name, engine=engine)) + node1.query(f"SYSTEM START MERGES {name}") + for i in range(5): data = [] # 5MB in total for i in range(5): @@ -523,8 +529,10 @@ def test_background_move(start_cluster, name, engine): # first (oldest) part was moved to external assert path.startswith("/external") + node1.query(f"SYSTEM START MERGES {name}") + finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -611,7 +619,7 @@ def test_start_stop_moves(start_cluster, name, engine): assert used_disks[0] == 'external' finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def get_path_for_part_from_part_log(node, table, part_name): @@ -699,7 +707,7 @@ def test_alter_move(start_cluster, name, engine): assert node1.query("SELECT COUNT() FROM {}".format(name)) == "4\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("volume_or_disk", [ @@ -748,7 +756,7 @@ def test_alter_move_half_of_partition(start_cluster, volume_or_disk): assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("volume_or_disk", [ @@ -792,7 +800,7 @@ def test_alter_double_move_partition(start_cluster, volume_or_disk): volume_or_disk=volume_or_disk)) finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def produce_alter_move(node, name): @@ -876,7 +884,7 @@ def test_concurrent_alter_move(start_cluster, name, engine): assert node1.query("SELECT 1") == "1\n" assert node1.query("SELECT COUNT() FROM {}".format(name)) == "500\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -929,7 +937,7 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): assert node1.query("SELECT 1") == "1\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -960,7 +968,7 @@ def test_detach_attach(start_cluster, name, engine): assert node1.query("SELECT count() FROM {}".format(name)).strip() == "5" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -1006,7 +1014,7 @@ def test_mutate_to_another_disk(start_cluster, name, engine): finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @pytest.mark.parametrize("name,engine", [ @@ -1064,7 +1072,7 @@ def test_concurrent_alter_modify(start_cluster, name, engine): assert node1.query("SELECT COUNT() FROM {}".format(name)) == "100\n" finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def test_simple_replication_and_moves(start_cluster): @@ -1131,7 +1139,7 @@ def test_simple_replication_and_moves(start_cluster): set(disks2) == set(["jbod1", "external"]) finally: for node in [node1, node2]: - node.query("DROP TABLE IF EXISTS replicated_table_for_moves") + node.query("DROP TABLE IF EXISTS replicated_table_for_moves SYNC") def test_download_appropriate_disk(start_cluster): @@ -1165,7 +1173,7 @@ def test_download_appropriate_disk(start_cluster): finally: for node in [node1, node2]: - node.query("DROP TABLE IF EXISTS replicated_table_for_download") + node.query("DROP TABLE IF EXISTS replicated_table_for_download SYNC") def test_rename(start_cluster): @@ -1202,9 +1210,9 @@ def test_rename(start_cluster): node1.query("SELECT COUNT() FROM default.renaming_table1") finally: - node1.query("DROP TABLE IF EXISTS default.renaming_table") - node1.query("DROP TABLE IF EXISTS default.renaming_table1") - node1.query("DROP TABLE IF EXISTS test.renaming_table2") + node1.query("DROP TABLE IF EXISTS default.renaming_table SYNC") + node1.query("DROP TABLE IF EXISTS default.renaming_table1 SYNC") + node1.query("DROP TABLE IF EXISTS test.renaming_table2 SYNC") def test_freeze(start_cluster): @@ -1238,7 +1246,7 @@ def test_freeze(start_cluster): node1.exec_in_container(["bash", "-c", "find /external/shadow -name '*.mrk2' | grep '.*'"]) finally: - node1.query("DROP TABLE IF EXISTS default.freezing_table") + node1.query("DROP TABLE IF EXISTS default.freezing_table SYNC") node1.exec_in_container(["rm", "-rf", "/jbod1/shadow", "/external/shadow"]) @@ -1282,7 +1290,7 @@ def test_kill_while_insert(start_cluster): finally: try: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") except: """ClickHouse may be inactive at this moment and we don't want to mask a meaningful exception.""" @@ -1343,7 +1351,7 @@ def test_move_while_merge(start_cluster): assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["2"] finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") def test_move_across_policies_does_not_work(start_cluster): @@ -1384,8 +1392,8 @@ def test_move_across_policies_does_not_work(start_cluster): assert node1.query("""SELECT * FROM {name}""".format(name=name)).splitlines() == ["1"] finally: - node1.query("DROP TABLE IF EXISTS {name}".format(name=name)) - node1.query("DROP TABLE IF EXISTS {name}2".format(name=name)) + node1.query(f"DROP TABLE IF EXISTS {name} SYNC") + node1.query(f"DROP TABLE IF EXISTS {name}2 SYNC") def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, parts_after_cmds): @@ -1420,7 +1428,7 @@ def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, pa assert len(parts) == parts_after_cmds finally: - node.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node.query(f"DROP TABLE IF EXISTS {name} SYNC") def _check_merges_are_working(node, storage_policy, volume, shall_work): @@ -1458,7 +1466,7 @@ def _check_merges_are_working(node, storage_policy, volume, shall_work): assert len(parts) == 1 if shall_work else created_parts finally: - node.query("DROP TABLE IF EXISTS {name}".format(name=name)) + node.query(f"DROP TABLE IF EXISTS {name} SYNC") def _get_prefer_not_to_merge_for_storage_policy(node, storage_policy): diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index ab44d0ebea0..d7f98d5cb77 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -537,14 +537,14 @@ def test_rabbitmq_big_message(rabbitmq_cluster): @pytest.mark.timeout(420) def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): NUM_CONSUMERS = 10 - NUM_QUEUES = 2 + NUM_QUEUES = 10 instance.query(''' CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'test_sharding', - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_num_consumers = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -617,7 +617,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): rabbitmq_exchange_name = 'combo', rabbitmq_queue_base = 'combo', rabbitmq_num_consumers = 2, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 5, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; ''') @@ -879,7 +879,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_queue_base = 'over', rabbitmq_exchange_type = 'direct', rabbitmq_num_consumers = 5, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_max_block_size = 10000, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', @@ -1722,7 +1722,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'consumer_reconnect', rabbitmq_num_consumers = 10, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; ''') diff --git a/tests/integration/test_system_flush_logs/__init__.py b/tests/integration/test_system_flush_logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py b/tests/integration/test_system_flush_logs/test.py similarity index 100% rename from tests/integration/test_SYSTEM_FLUSH_LOGS/test.py rename to tests/integration/test_system_flush_logs/test.py diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py index 7f5bce97805..b159e8b4cf3 100644 --- a/tests/integration/test_system_queries/test.py +++ b/tests/integration/test_system_queries/test.py @@ -107,7 +107,7 @@ def test_RELOAD_CONFIG_AND_MACROS(started_cluster): assert TSV(instance.query("select * from system.macros")) == TSV("instance\tch1\nmac\tro\n") -def test_SYSTEM_FLUSH_LOGS(started_cluster): +def test_system_flush_logs(started_cluster): instance = cluster.instances['ch1'] instance.query(''' SET log_queries = 0; diff --git a/tests/queries/0_stateless/00273_quantiles.reference b/tests/queries/0_stateless/00273_quantiles.reference index d75d11191f4..616e06841e4 100644 --- a/tests/queries/0_stateless/00273_quantiles.reference +++ b/tests/queries/0_stateless/00273_quantiles.reference @@ -6,17 +6,17 @@ [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] [0,0.50100005,9.51,49.55,99.6,199.7,299.8,399.9,500,600.1,700.2,800.3,900.4,950.45,990.49,999.499,1000] [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] -1 333334 [699140.3,835642,967430.8] [699999,833333,966666] +1 333334 [699144.2,835663,967429.2] [699999,833333,966666] 2 266667 [426549.5,536255.5,638957.6] [426665,533332,639999] -3 114285 [296938.5,342335,388777.5] [297142,342856,388570] +3 114285 [296938,342324,388778] [297142,342856,388570] 4 63492 [228370.2,254019.5,279351.4] [228571,253968,279364] 5 40404 [185603.4,202009,218107] [185858,202020,218181] -6 27972 [156598.6,167864,179118.40000000002] [156643,167832,179020] -7 20513 [135401,143553.5,151792.5] [135384,143589,151794] +6 27972 [156598.7,167866,179118.3] [156643,167832,179020] +7 20513 [135400.8,143550,151792.6] [135384,143589,151794] 8 15686 [119239.20000000001,125463,131772.40000000002] [119215,125490,131764] -9 12384 [106509.79999999999,111538,116415.8] [106501,111455,116408] -10 10025 [96223.6,100347,104288.6] [96240,100250,104260] -11 8282 [87732.70000000001,91035,94408.6] [87784,91097,94409] +9 12384 [106510.20000000001,111539,116415.7] [106501,111455,116408] +10 10025 [96223.2,100346,104288.7] [96240,100250,104260] +11 8282 [87732.8,91036,94410.20000000001] [87784,91097,94409] 12 6957 [80694.6,83477,86259.4] [80694,83477,86260] 13 5925 [74666.40000000001,77036,79405.6] [74666,77036,79406] 14 5109 [69475.8,71519,73562.2] [69475,71519,73563] diff --git a/tests/queries/0_stateless/00932_geohash_support.sql b/tests/queries/0_stateless/00932_geohash_support.sql index a477332b532..aeed72176b9 100644 --- a/tests/queries/0_stateless/00932_geohash_support.sql +++ b/tests/queries/0_stateless/00932_geohash_support.sql @@ -45,7 +45,7 @@ select 12 as p, geohashEncode(longitude, latitude, p) as actual, if(actual = enc -- Here results are floats, and hence may not be compared for equality directly. -- We select all values that are off by some reasonable value: --- each byte of encoded string provides 5 bits of precison, (roughly 2.5 for lon and lat) +-- each byte of encoded string provides 5 bits of precision, (roughly 2.5 for lon and lat) -- each bit of precision divides value range by 2. -- hence max error is roughly value range 2.5 times divided by 2 for each precision bit. -- initial value range is [-90..90] for latitude and [-180..180] for longitude. diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 9f4bfb2c436..173bf44e1f1 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -10,18 +10,18 @@ $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01107.mt (n UInt64) ENGINE=MergeTree() $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(3) FROM numbers(5)" & sleep 1 -$CLICKHOUSE_CLIENT -q "DETACH TABLE test_01107.mt" +$CLICKHOUSE_CLIENT -q "DETACH TABLE test_01107.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" 2>&1 | grep -F "Code: 57" > /dev/null && echo "OK" -$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK" +$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK" wait $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" -$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" +$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "ATTACH DATABASE test_01107" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" & sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" && sleep 1 && echo "dropped" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 && sleep 1 && echo "dropped" wait diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index d23be44e784..d220491d152 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -36,9 +36,10 @@ $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_3.mt_tmp" $CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_3" -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01114_2.mt UUID '00001114-0000-4000-8000-000000000002' (n UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY (n % 5)" -$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" -$CLICKHOUSE_CLIENT -q "SELECT name, uuid, create_table_query FROM system.tables WHERE database='test_01114_2'" +explicit_uuid=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01114_2.mt UUID '$explicit_uuid' (n UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY (n % 5)" +$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" +$CLICKHOUSE_CLIENT -q "SELECT name, uuid, create_table_query FROM system.tables WHERE database='test_01114_2'" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" $CLICKHOUSE_CLIENT -q "SELECT count(col), sum(col) FROM (SELECT n + sleepEachRow(1.5) AS col FROM test_01114_1.mt)" & # 30s, result: 20, 190 @@ -55,18 +56,18 @@ $CLICKHOUSE_CLIENT -q "EXCHANGE TABLES test_01114_1.mt AND test_01114_2.mt" $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" uuid_mt1=$($CLICKHOUSE_CLIENT -q "SELECT uuid FROM system.tables WHERE database='test_01114_1' AND name='mt'") $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_1.mt" | sed "s/$uuid_mt1/00001114-0000-4000-8000-000000000001/g" -$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" +$CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE TABLE test_01114_2.mt" | sed "s/$explicit_uuid/00001114-0000-4000-8000-000000000002/g" -$CLICKHOUSE_CLIENT -q "DROP TABLE test_01114_1.mt" +$CLICKHOUSE_CLIENT -q "DROP TABLE test_01114_1.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01114_1.mt (s String) ENGINE=Log()" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01114_1.mt SELECT 's' || toString(number) FROM numbers(5)" $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" # result: 5 $CLICKHOUSE_CLIENT -q "SELECT tuple(s, sleepEachRow(3)) FROM test_01114_1.mt" > /dev/null & # 15s sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" && echo "dropped" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" --database_atomic_wait_for_drop_and_detach_synchronously=0 && echo "dropped" wait # for INSERT $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01114_2.mt" # result: 30, 435 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_2" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_2" --database_atomic_wait_for_drop_and_detach_synchronously=0 diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index 0ee583a7265..319b537e84b 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -49,4 +49,4 @@ $CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(query_duration_ms) < $max_time_ms $CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" -$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" +$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" --database_atomic_wait_for_drop_and_detach_synchronously=0 diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh index b14defd672a..5333d0b4b0b 100755 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Regression for MemoryTracker that had been incorrectly accounted -# (it was reseted before deallocation) +# (it was reset before deallocation) # # For this will be used: # - two-level group by diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql index babc8457bc0..b8b86384bc4 100644 --- a/tests/queries/0_stateless/01322_student_ttest.sql +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS student_ttest; CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (0.88854,-2.90702), (-5.76966,3.61651), (6.76618,4.27458), (3.55546,4.82133), (-9.76948,9.59483), (4.92323,1.00424), (-0.36352,2.04147), (0.97018,-3.58214), (4.61656,6.59543), (-6.78292,-1.00532), (4.02008,-3.59794), (12.41838,-2.82434), (5.14417,-3.13194), (3.86836,9.90977), (-1.26199,0.523), (12.44106,4.62779), (3.28349,-2.56872), (1.77261,2.25807), (-8.94748,1.04044), (-1.01449,-2.35744), (-1.26377,10.81531), (6.79682,-9.68469), (6.32333,3.80885), (-8.21214,12.70435), (-1.68565,-6.01112), (9.7557,1.89065), (3.66694,5.08892), (1.39967,3.45254), (-5.52035,11.58151), (-10.95601,0.85035), (0.93877,8.38397), (1.45933,1.17169), (-5.40551,4.74621), (-0.83857,-1.66614), (8.50794,4.2414), (-6.68686,1.68765), (5.03099,1.85223), (1.56251,9.10111), (4.17381,-2.38085), (-2.92644,-14.79595), (5.11068,-3.8938), (2.09617,-3.41864), (11.7787,-3.15282), (6.50336,-0.56684), (0.62098,12.87997), (-7.97121,6.89115), (3.81902,12.921), (0.33151,-7.94908), (10.68584,2.45687), (0.56007,2.14957), (-7.38621,7.55081), (5.05882,-3.71534), (2.34616,-2.41064), (11.3806,-0.80734), (5.95276,-4.75651), (-3.01429,2.05241), (5.98169,-5.44523), (0.96985,-2.75054), (-1.15932,-13.00131), (2.11547,-2.74451), (2.49668,-1.39004), (-12.49569,-3.02854), (-4.94667,7.65112), (-3.64215,1.1245), (-8.35595,6.74117), (3.211,-0.75777), (2.33805,8.93451), (2.38608,-8.85559), (-3.2862,-0.36405), (-0.80454,4.02742), (-0.53483,6.88718), (10.66445,-1.05124), (-0.37619,3.04085), (0.48246,3.32368), (7.41919,1.147), (0.42414,3.41554), (-2.32335,-3.47851), (-0.70223,-0.47684), (-5.9332,-0.55605), (-1.20561,-0.17006), (3.39865,2.26218), (9.61739,12.45494), (-0.78651,-1.84097), (-4.00256,1.64934), (-7.99646,-7.07496), (8.72923,-9.99462), (0.71859,6.09954), (-1.62726,-1.05319), (5.11234,3.04757), (-0.95625,0.93899), (-3.75573,-4.63243), (1.03141,-7.43322), (-3.33588,-7.298), (1.51804,-6.59016), (-3.30935,-6.11649), (-1.97507,0.56682), (4.06456,2.00661), (3.27195,-2.79814), (-7.81761,2.84482), (-3.81785,3.65348), (-4.18311,-4.22807), (-11.33313,-4.54336), (-0.25221,-3.63343), (7.2514,2.96878), (5.30301,6.11661), (2.46762,-1.70919), (4.22716,-4.71133), (0.33916,6.09652), (9.7638,-6.83454), (-7.58684,0.18006), (-4.09888,1.51676), (4.26617,-5.31646), (-0.56744,-3.21215), (4.65125,-5.07599), (-1.30301,-2.36591), (4.53771,3.55724), (9.96929,4.8904), (3.72939,-3.22586), (-2.29818,-1.74928), (3.09417,5.73458), (0.82251,1.41188), (5.29975,2.86255), (2.8685,2.90179), (-5.73321,-2.19949), (-1.85651,1.72727), (-1.07984,1.76939), (9.78342,-0.12848), (-13.49652,-0.52), (3.68791,3.48333), (1.9998,7.8262), (1.11674,0.09099), (9.43869,7.77017), (4.07029,9.49484), (5.32715,1.42825), (7.16504,1.99624), (6.66096,4.00419), (-5.7111,1.07925), (-0.38575,-0.09987), (4.49165,-5.48733), (-3.36489,-1.83517), (7.71814,2.38059), (-1.58966,1.42075), (-1.61063,-1.11968), (-0.91602,-6.46035), (0.73459,7.66576), (-3.24463,4.6307), (6.3947,5.55989), (-2.77845,3.16684), (4.45899,5.07671), (-8.84186,-10.20566), (2.62276,-4.73386), (1.774,1.28353), (4.3692,6.75679), (0.05942,12.09895), (-1.44042,7.0049), (-2.53594,7.16156), (-2.24752,-0.64311), (4.98874,-0.66747), (4.05434,3.99996), (-2.56483,9.07298), (-6.79286,-4.60971), (-2.06165,0.70744), (-0.26056,2.56774), (1.89567,9.32424), (-3.15145,3.95087), (-7.31321,7.11372), (0.28936,-0.89284), (-0.63111,8.6155), (0.22611,-0.14141), (-9.3377,-4.86319), (-5.76638,-6.95801), (3.87306,4.44883), (6.7011,4.6156), (9.03915,-2.3579), (-1.21835,-5.1186), (0.82892,8.12819), (2.80656,2.78392), (-1.34746,-4.30221), (-1.99912,-1.47506), (0.6036,6.8598), (-3.46117,0.47636), (5.23732,0.95383), (-1.86702,7.79779), (-5.86115,-2.61767), (6.48523,-10.5087), (-7.40158,-2.74299), (-1.38913,3.87369), (4.94613,-1.07093), (-2.07818,4.98864), (2.39808,-7.50772), (4.89238,6.41316), (4.39481,1.39061), (5.20425,-3.1747), (13.62598,-2.13621), (-2.86293,-0.02203), (-3.62396,0.89025), (-4.28695,-5.87746), (4.66425,3.60026), (2.20871,-0.23178), (1.60382,-2.1897), (-9.87024,-5.85101), (-7.37302,-1.6053), (-4.17814,3.6184), (2.5148,-8.53795), (3.21708,-0.35987), (-11.48089,2.15301), (1.19821,-6.60692), (-0.07436,9.54341), (-1.10652,1.11511), (4.03395,2.94025), (-4.35883,12.05657), (2.04013,3.75156), (0.52264,7.95597), (8.14004,-0.99449), (-8.86949,0.90597), (-0.35807,-7.90627), (-10.71113,3.50863), (-2.13755,-1.47493), (0.50715,4.11671), (6.30826,10.06325), (2.37527,-1.06059), (0.20872,-1.37737), (-5.85729,-0.42542), (-4.97217,-3.90267), (-9.78434,9.35037), (-1.53277,-7.91219), (0.14827,-4.69945), (-1.053,3.63776), (1.74558,3.46492), (11.17194,2.84518), (9.35487,-3.04301), (-9.17209,8.82764), (10.41814,7.80134), (7.41206,7.87755), (3.71775,7.01035), (-2.04674,2.43271), (6.18037,11.36418), (5.6383,-6.92659), (-0.90058,5.95541), (-1.27073,3.59436), (-2.3473,5.18429), (-8.44271,4.20225), (2.75551,0.5029), (-1.15521,4.03074), (4.08722,5.23152), (-1.70399,10.65409), (7.24114,-0.69845), (-8.43976,11.70096), (-1.53052,5.80692), (-0.00526,-8.1819), (-4.04813,4.31485), (-2.84299,5.7227), (-5.201,5.67398), (7.75774,-1.75826), (-2.85791,7.54164), (-3.86071,-1.79026), (-1.80029,-1.7395), (-5.26015,5.65042), (-3.158,0.38765), (7.71014,-4.64719), (-4.84866,-10.22048), (-8.38785,-2.05447), (7.67021,-2.43441), (4.96521,-5.38551), (-0.40919,5.47764), (-3.25711,8.26637), (3.07685,-3.6421), (2.89376,-11.66269), (-10.47331,3.972), (-3.48942,5.46642), (1.13906,-3.72304), (-8.57454,5.75251), (-3.38963,5.12841), (-2.3195,0.59067), (-1.60694,5.21138), (-5.57406,-4.58702), (-0.93075,-8.737), (-11.76579,-2.12737), (10.68283,0.22888), (8.74324,-1.46448), (7.66409,2.40311), (4.76715,-5.21814), (0.44539,13.94749), (-1.35941,-2.77448), (4.18849,-3.7867), (-6.17097,3.4954), (0.27977,3.12586), (-1.45006,-7.01485), (-4.81694,-3.20727), (-3.0297,6.31415), (0.02145,2.37521), (2.46883,8.13787), (9.60317,2.15956), (-9.93898,-0.40842), (1.05549,-7.27283), (5.55366,4.27575), (-3.80722,-2.89126), (-4.18851,6.84344), (1.00351,7.0869), (3.11385,-5.18837), (-5.17623,2.67648), (-3.18396,-6.57021), (-6.65302,0.60429), (-0.50832,-1.04921), (-4.04375,7.12873), (4.52707,1.68973), (6.63124,-2.58404), (-3.72082,-3.83114), (5.79825,-7.26546), (-2.0158,-5.07153), (-2.78369,-0.80395), (-1.91821,2.09455), (6.31714,4.33374), (-1.80869,8.54335), (8.55586,0.80566), (2.40826,-8.38085), (-8.46361,7.54812), (5.04452,8.78007), (-0.84665,1.5857), (2.30903,8.43855), (-3.71837,-1.90846), (-0.69419,-1.2434), (3.6733,7.16172), (-1.96098,-3.44129), (2.36747,-6.37542), (-12.03622,-4.99486), (4.38481,4.99033), (2.93955,-1.83734), (2.16804,-2.83289), (-0.08218,-4.13997), (-3.97934,1.40163), (-7.43985,8.57867), (0.91666,-1.87639), (7.23432,3.41667), (-6.13303,6.31762), (-10.23217,1.58473), (-6.21681,1.63625), (-0.80934,-6.93618), (0.17914,3.58046), (2.13338,-6.8097), (6.97656,4.69978), (6.90455,-1.72912), (6.25943,5.29491), (-6.04019,-1.63062), (-7.30909,5.83818), (1.4589,17.0769), (12.00208,4.54301), (2.22457,-1.33801), (-2.45912,5.64339), (-6.92213,1.26913), (4.05547,-1.01553), (0.04709,4.8316), (-7.70952,3.08635), (-1.47883,-2.27738), (1.3701,-1.13761), (-4.92928,10.08698), (-2.75872,5.33827), (-0.09178,2.84345), (2.62642,-1.51132), (-1.14623,13.46078), (2.76609,8.58965), (4.94404,-2.36683), (-7.01764,-1.8217), (-10.91568,1.96981), (-2.49738,2.31718), (0.73576,3.66493), (2.25436,1.93104), (-1.72956,5.20332), (2.41054,3.20519), (5.72149,3.34631), (-6.41371,7.0087), (3.38217,-7.96126), (1.24133,-0.62182), (10.03634,-4.65227), (-2.37303,10.6572), (-1.35543,4.50891), (-1.4387,9.74298), (-4.0976,3.85707), (-0.82501,6.41144), (-1.93498,1.48649), (5.59955,2.28076), (5.46656,2.75342), (2.43568,-5.40401), (-0.23926,7.11389), (-4.9945,5.74368), (-4.96655,6.78345), (-0.59258,3.83773), (2.02497,0.70959), (0.67583,0.57434), (3.16522,1.5888), (-1.9673,3.94889), (-6.75319,5.8234), (-6.69723,7.78366), (0.81148,9.08354), (4.44531,-7.99182), (-4.43522,-2.77033), (-5.28602,-10.29342), (-3.58829,1.76251), (-7.97395,2.09266), (-2.84891,4.20614), (-3.95112,-3.63064), (3.54945,-2.17794), (12.12376,-2.66225), (-3.12347,-2.74707), (3.65209,-1.93431), (9.34031,1.38629), (-0.26348,4.12816), (-5.23968,-1.58902), (2.22336,-5.08864), (-10.70405,-2.30491), (-4.41319,2.64605), (-5.94912,1.16158), (1.8147,2.63534), (7.69287,1.4956), (9.46125,-4.60768), (4.72497,0.60771), (-0.57565,3.29549), (-1.12303,-1.42592), (2.90272,0.8883), (-4.4584,-1.10612), (4.28819,-2.57296), (11.64512,5.88085), (-1.80395,7.40745), (2.51605,13.48116), (-3.18439,5.53539), (-0.70213,-1.46014), (-7.68383,3.73304), (-8.32268,3.5435), (-8.71115,-3.89151), (9.96933,4.16265), (0.95675,2.32663), (3.35114,5.31735), (-2.66008,6.33485), (7.75456,2.1339), (0.73568,0.82708), (0.3483,-2.95155), (-1.09203,-6.76019), (-7.76963,-4.20179), (5.81902,8.78354), (-3.41424,1.41863), (-0.39209,7.65689), (4.67608,-6.52601), (0.68753,-4.4426), (5.17179,-4.49483), (4.98983,-3.91479), (-0.12659,-2.84562), (3.25267,2.58974), (1.50184,2.24424), (2.94507,-4.65846), (-0.42333,8.4062), (-3.66227,8.20262), (8.90812,-8.63752), (4.74411,4.97966), (2.22018,-0.35563), (-2.07976,-4.72116), (4.8711,-2.95997), (0.5023,2.73959), (6.31569,-0.23956), (-4.36903,10.13915), (3.82146,11.83775), (-6.99477,-2.50332), (3.61225,-0.58181), (14.69335,-7.62836), (0.58368,2.26478), (4.65341,-3.50179), (-3.14272,-2.08023), (2.67048,4.07256), (4.64963,-1.40826), (-2.70828,-2.33644), (1.42923,3.00197), (5.84498,4.23668), (-4.76568,-2.24647), (0.19907,1.0445), (1.67486,-0.31901), (5.32145,8.62657), (-8.03477,3.92817), (3.46776,0.08462), (4.66374,10.15884), (-5.37394,0.4113), (5.39045,4.45847), (-1.44756,5.82941), (-1.64419,6.59202), (3.39699,-3.73441), (-2.94659,-5.86969), (-2.38437,-4.56543), (-0.23958,-1.32636), (6.88389,-0.17884), (-2.7172,-3.56181), (-1.53419,-0.66932), (7.38841,6.87538), (-5.44178,0.73527), (-0.89287,-0.24177), (2.93546,-0.8657), (-0.26901,-0.22977), (-4.70044,1.02095), (2.25846,6.16311), (-9.28813,-5.68027), (6.04268,-3.7619), (4.41693,4.22959), (1.75714,-1.5249); SELECT '-2.610898982580138', '0.00916587538237954'; -SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -15,5 +15,5 @@ DROP TABLE IF EXISTS student_ttest; CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); SELECT '-28.740781574102936', '7.667329672103986e-133'; -SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 5a5b52ab612..cce65c28bd8 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -3,19 +3,19 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (27.5,27.1), (21.0,22.0), (19.0,20.8), (23.6,23.4), (17.0,23.4), (17.9,23.5), (16.9,25.8), (20.1,22.0), (21.9,24.8), (22.6,20.2), (23.1,21.9), (19.6,22.1), (19.0,22.9), (21.7,20.5), (21.4,24.4); SELECT '0.021378001462867'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (30.02,29.89), (29.99,29.93), (30.11,29.72), (29.97,29.98), (30.01,30.02), (29.99,29.98); SELECT '0.090773324285671'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (0.010268,0.159258), (0.000167,0.136278), (0.000167,0.122389); SELECT '0.00339907162713746'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -24,7 +24,7 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); SELECT '-0.5028215369186904', '0.6152361677168877'; -SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -33,5 +33,5 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); SELECT '14.971190998235835', '5.898143508382202e-44'; -SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference new file mode 100644 index 00000000000..420bb17ae8b --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -0,0 +1,10 @@ +===http=== +{"total spans":"4","unique spans":"4","unique non-zero parent spans":"2"} +{"initial query spans with proper parent":"1"} +{"unique non-empty tracestate values":"1"} +===native=== +{"total spans":"2","unique spans":"2","unique non-zero parent spans":"2"} +{"initial query spans with proper parent":"1"} +{"unique non-empty tracestate values":"1"} +===sampled=== +OK diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh new file mode 100755 index 00000000000..fe52b2b2032 --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -ue + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +function check_log +{ +${CLICKHOUSE_CLIENT} --format=JSONEachRow -nq " +system flush logs; + +-- Check the number of query spans with given trace id, to verify it was +-- propagated. +select count(*) "'"'"total spans"'"'", + uniqExact(span_id) "'"'"unique spans"'"'", + uniqExactIf(parent_span_id, parent_span_id != 0) + "'"'"unique non-zero parent spans"'"'" + from system.opentelemetry_span_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + ; + +-- Also check that the initial query span in ClickHouse has proper parent span. +select count(*) "'"'"initial query spans with proper parent"'"'" + from + (select *, attribute_name, attribute_value + from system.opentelemetry_span_log + array join attribute.names as attribute_name, + attribute.values as attribute_value) o + join system.query_log on query_id = o.attribute_value + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + and parent_span_id = reinterpretAsUInt64(unhex('73')) + and o.attribute_name = 'clickhouse.query_id' + and is_initial_query + and type = 'QueryFinish' + ; + +-- Check that the tracestate header was propagated. It must have exactly the +-- same non-empty value for all 'query' spans in this trace. +select uniqExact(value) "'"'"unique non-empty tracestate values"'"'" + from system.opentelemetry_span_log + array join attribute.names as name, attribute.values as value + where + trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + and name = 'clickhouse.tracestate' + and length(value) > 0 + ; +" +} + +# Generate some random trace id so that the prevous runs of the test do not interfere. +echo "===http===" +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +# Check that the HTTP traceparent is read, and then passed through `remote` +# table function. We expect 4 queries -- one initial, one SELECT and two +# DESC TABLE. Two DESC TABLE instead of one looks like a bug, see the issue: +# https://github.com/ClickHouse/ClickHouse/issues/14228 +${CLICKHOUSE_CURL} \ + --header "traceparent: 00-$trace_id-0000000000000073-01" \ + --header "tracestate: some custom state" "http://localhost:8123/" \ + --get \ + --data-urlencode "query=select 1 from remote('127.0.0.2', system, one) format Null" + +check_log + +# With another trace id, check that clickhouse-client accepts traceparent, and +# that it is passed through URL table function. We expect two query spans, one +# for the initial query, and one for the HTTP query. +echo "===native===" +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +${CLICKHOUSE_CLIENT} \ + --opentelemetry-traceparent "00-$trace_id-0000000000000073-01" \ + --opentelemetry-tracestate "another custom state" \ + --query "select * from url('http://127.0.0.2:8123/?query=select%201%20format%20Null', CSV, 'a int')" + +check_log + +# Test sampled tracing. The traces should be started with the specified +# probability, only for initial queries. +echo "===sampled===" +query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +for i in {1..200} +do + ${CLICKHOUSE_CLIENT} \ + --opentelemetry_start_trace_probability=0.1 \ + --query_id "$query_id-$i" \ + --query "select 1 from remote('127.0.0.2', system, one) format Null" \ + & + + # clickhouse-client is slow to start (initialization of DateLUT), so run + # several clients in parallel, but not too many. + if [[ $((i % 10)) -eq 0 ]] + then + wait + fi +done +wait + +${CLICKHOUSE_CLIENT} -q "system flush logs" +${CLICKHOUSE_CLIENT} -q " + with count(*) as c + -- expect 200 * 0.1 = 20 sampled events on average + select if(c > 5 and c < 35, 'OK', 'fail: ' || toString(c)) + from system.opentelemetry_span_log + array join attribute.names as name, attribute.values as value + where name = 'clickhouse.query_id' + and operation_name = 'query' + and parent_span_id = 0 -- only account for the initial queries + and value like '$query_id-%' + ; +" diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.expect b/tests/queries/0_stateless/01526_client_start_and_exit.expect new file mode 100755 index 00000000000..003439ffa54 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.expect @@ -0,0 +1,12 @@ +#!/usr/bin/expect -f + +log_user 1 +set timeout 5 +match_max 100000 + +if ![info exists env(CLICKHOUSE_PORT_TCP)] {set env(CLICKHOUSE_PORT_TCP) 9000} + +spawn bash -c "clickhouse-client --port $env(CLICKHOUSE_PORT_TCP) && echo $?" +expect ":) " +send -- "\4" +expect eof diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.reference b/tests/queries/0_stateless/01526_client_start_and_exit.reference new file mode 100644 index 00000000000..e3e2e7b22af --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.reference @@ -0,0 +1 @@ +Loaded 10000 queries. diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh new file mode 100755 index 00000000000..c179be79d03 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# Create a huge amount of tables, so Suggest will take a time to load +${CLICKHOUSE_CLIENT} -q "SELECT 'CREATE TABLE test_' || hex(randomPrintableASCII(40)) || '(x UInt8) Engine=Memory;' FROM numbers(10000)" --format=TSVRaw | ${CLICKHOUSE_BENCHMARK} -c32 -i 10000 -d 0 2>&1 | grep -F 'Loaded 10000 queries' + +function stress() +{ + while true; do + "${CURDIR}"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' + done +} + +export CURDIR +export -f stress + +for _ in {1..10}; do + timeout 3 bash -c stress & +done + +wait diff --git a/tests/queries/0_stateless/01526_initial_query_id.reference b/tests/queries/0_stateless/01526_initial_query_id.reference new file mode 100644 index 00000000000..e8d2c31aa17 --- /dev/null +++ b/tests/queries/0_stateless/01526_initial_query_id.reference @@ -0,0 +1,2 @@ +1 1 +2 1 diff --git a/tests/queries/0_stateless/01526_initial_query_id.sh b/tests/queries/0_stateless/01526_initial_query_id.sh new file mode 100755 index 00000000000..c5459625023 --- /dev/null +++ b/tests/queries/0_stateless/01526_initial_query_id.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ue + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +${CLICKHOUSE_CLIENT} -q "select 1 format Null" "--query_id=$query_id" + +${CLICKHOUSE_CURL} \ + --header "X-ClickHouse-Query-Id: $query_id" \ + "http://localhost:8123/" \ + --get \ + --data-urlencode "query=select 1 format Null" + +${CLICKHOUSE_CLIENT} -n -q " +system flush logs; +select interface, initial_query_id = query_id + from system.query_log + where query_id = '$query_id' and type = 'QueryFinish' + order by interface + ; +" + diff --git a/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.reference b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh new file mode 100755 index 00000000000..f341fbcdd9b --- /dev/null +++ b/tests/queries/0_stateless/01532_clickhouse_local_tmp_folder.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# in case when clickhouse-local can't use temp folder it will try to create +# temporary subfolder in the current dir +TMP=/non-existent-folder-12123 ${CLICKHOUSE_LOCAL} -q 'SELECT 1' diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.reference b/tests/queries/0_stateless/01532_min_max_with_modifiers.reference new file mode 100644 index 00000000000..9e1683ca6f9 --- /dev/null +++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.reference @@ -0,0 +1,20 @@ +totals +1 1 1 +2 2 2 +3 3 3 + +0 1 3 +rollup +1 1 1 +2 2 2 +3 3 3 +0 1 3 +cube +1 1 1 +2 2 2 +3 3 3 +0 1 3 +======= +1 1 2 1 +2 2 3 1 +0 1 3 2 diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.sql b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql new file mode 100644 index 00000000000..0c8651c0f01 --- /dev/null +++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql @@ -0,0 +1,18 @@ +SELECT 'totals'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH TOTALS; +SELECT 'rollup'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH ROLLUP; +SELECT 'cube'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH CUBE; +SELECT '======='; + +SELECT + x, + min(x) AS lower, + max(x) + 1 AS upper, + upper - lower AS range +FROM +( + SELECT arrayJoin([1, 2]) AS x +) +GROUP BY x WITH ROLLUP; diff --git a/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql new file mode 100644 index 00000000000..4f32576e86b --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS bug_13492; + +CREATE TABLE bug_13492 (`d` DateTime) ENGINE = MergeTree +PARTITION BY toYYYYMMDD(d) ORDER BY tuple(); + +INSERT INTO bug_13492 SELECT addDays(now(), number) FROM numbers(100); + +SET max_threads = 5; + +SELECT DISTINCT 1 FROM bug_13492, numbers(1) n; + +SET max_threads = 2; + +SELECT DISTINCT 1 FROM bug_13492, numbers(1) n; + +DROP TABLE bug_13492; diff --git a/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference b/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference new file mode 100644 index 00000000000..e02acad09d6 --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference @@ -0,0 +1,4 @@ +442d3ff4-842a-45bb-8b02-b616122c0dc6 +05fe40cb-1d0c-45b0-8e60-8e311c2463f1 +2fc89389-4728-4b30-9e51-b5bc3ad215f6 +10000 diff --git a/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql b/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql new file mode 100644 index 00000000000..926739d3f58 --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS bug_14144; + +CREATE TABLE bug_14144 +( meta_source_req_uuid Nullable(UUID), + a Int64, + meta_source_type String +) +ENGINE = MergeTree +ORDER BY a; + +INSERT INTO bug_14144 SELECT cast(toUUID('442d3ff4-842a-45bb-8b02-b616122c0dc6'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +INSERT INTO bug_14144 SELECT cast(toUUIDOrZero('2fc89389-4728-4b30-9e51-b5bc3ad215f6'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +INSERT INTO bug_14144 SELECT cast(toUUIDOrNull('05fe40cb-1d0c-45b0-8e60-8e311c2463f1'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +SELECT DISTINCT meta_source_req_uuid +FROM bug_14144 +WHERE meta_source_type = 'missing' +ORDER BY meta_source_req_uuid ASC; + +TRUNCATE TABLE bug_14144; + +INSERT INTO bug_14144 SELECT generateUUIDv4(), number, 'missing' FROM numbers(10000); + +SELECT COUNT() FROM ( + SELECT DISTINCT meta_source_req_uuid + FROM bug_14144 + WHERE meta_source_type = 'missing' + ORDER BY meta_source_req_uuid ASC + LIMIT 100000 +); + +DROP TABLE bug_14144; + + + + diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference new file mode 100644 index 00000000000..231c72269ca --- /dev/null +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference @@ -0,0 +1 @@ +3998 diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql b/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql new file mode 100644 index 00000000000..c75e5dd501f --- /dev/null +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql @@ -0,0 +1 @@ +SELECT quantileDeterministic(number, sipHash64(number)) FROM remote('127.0.0.{1,2}', numbers(8193)); diff --git a/tests/queries/0_stateless/01534_lambda_array_join.reference b/tests/queries/0_stateless/01534_lambda_array_join.reference new file mode 100644 index 00000000000..78e098be306 --- /dev/null +++ b/tests/queries/0_stateless/01534_lambda_array_join.reference @@ -0,0 +1,3 @@ +[NULL] +[1,1] +\N 70 diff --git a/tests/queries/0_stateless/01534_lambda_array_join.sql b/tests/queries/0_stateless/01534_lambda_array_join.sql new file mode 100644 index 00000000000..aee9dd1411a --- /dev/null +++ b/tests/queries/0_stateless/01534_lambda_array_join.sql @@ -0,0 +1,12 @@ +SELECT arrayMap(x -> concat(x, concat(arrayJoin([1]), x, NULL), ''), [1]); +SELECT arrayMap(x -> arrayJoin([1]), [1, 2]); + +SELECT + arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomString(range(randomString(1048577), NULL), arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomString(range(NULL), 65537), 255)), range(1))), substring(randomString(NULL), x + 7), '257'), 1025)), range(7))) AS byte, + count() AS c + FROM numbers(10) + GROUP BY + arrayMap(x -> reinterpretAsUInt8(substring(randomString(randomString(range(randomString(255), NULL)), NULL))), range(3)), + randomString(range(randomString(1048577), NULL), NULL), + byte + ORDER BY byte ASC; diff --git a/tests/queries/0_stateless/01536_fuzz_cast.reference b/tests/queries/0_stateless/01536_fuzz_cast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01536_fuzz_cast.sql b/tests/queries/0_stateless/01536_fuzz_cast.sql new file mode 100644 index 00000000000..436d76b5c4c --- /dev/null +++ b/tests/queries/0_stateless/01536_fuzz_cast.sql @@ -0,0 +1 @@ +SELECT CAST(arrayJoin([NULL, '', '', NULL, '', NULL, '01.02.2017 03:04\005GMT', '', NULL, '01/02/2017 03:04:05 MSK01/02/\0017 03:04:05 MSK', '', NULL, '03/04/201903/04/201903/04/\001903/04/2019']), 'Enum8(\'a\' = 1, \'b\' = 2)') AS x; -- { serverError 349 } diff --git a/tests/queries/0_stateless/01537_fuzz_count_equal.reference b/tests/queries/0_stateless/01537_fuzz_count_equal.reference new file mode 100644 index 00000000000..dec7d2fabd2 --- /dev/null +++ b/tests/queries/0_stateless/01537_fuzz_count_equal.reference @@ -0,0 +1 @@ +\N diff --git a/tests/queries/0_stateless/01537_fuzz_count_equal.sql b/tests/queries/0_stateless/01537_fuzz_count_equal.sql new file mode 100644 index 00000000000..fde3fe19124 --- /dev/null +++ b/tests/queries/0_stateless/01537_fuzz_count_equal.sql @@ -0,0 +1 @@ +SELECT DISTINCT NULL = countEqual(materialize([arrayJoin([NULL, NULL, NULL]), NULL AS x, arrayJoin([255, 1025, NULL, NULL]), arrayJoin([2, 1048576, NULL, NULL])]), materialize(x)); diff --git a/tests/queries/0_stateless/01538_fuzz_aggregate.reference b/tests/queries/0_stateless/01538_fuzz_aggregate.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01538_fuzz_aggregate.sql b/tests/queries/0_stateless/01538_fuzz_aggregate.sql new file mode 100644 index 00000000000..13dadabda63 --- /dev/null +++ b/tests/queries/0_stateless/01538_fuzz_aggregate.sql @@ -0,0 +1,10 @@ +SELECT + count(), + sum(ns) +FROM +( + SELECT intDiv(number, NULL) AS k + FROM system.numbers_mt + GROUP BY k +) +ARRAY JOIN ns; -- { serverError 47 } diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.reference b/tests/queries/0_stateless/01542_dictionary_load_exception_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh new file mode 100755 index 00000000000..8782faac003 --- /dev/null +++ b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE database_for_dict" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS database_for_dict.table_for_dict" +$CLICKHOUSE_CLIENT --query "CREATE TABLE database_for_dict.table_for_dict (key_column UInt64, second_column UInt64, third_column String) ENGINE = MergeTree() ORDER BY key_column" +$CLICKHOUSE_CLIENT --query "INSERT INTO database_for_dict.table_for_dict VALUES (100500, 10000000, 'Hello world')" + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ordinary_db" +$CLICKHOUSE_CLIENT --query "CREATE DICTIONARY ordinary_db.dict1 ( key_column UInt64 DEFAULT 0, second_column UInt64 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(max_result_bytes=1)" + +function dict_get_thread() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + done +} + +export -f dict_get_thread; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 1fe199be48f..9249fbc0411 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + export CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:="test"} export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} -[ -n "$CLICKHOUSE_CONFIG_CLIENT" ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " -[ -n "${CLICKHOUSE_HOST}" ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " -[ -n "${CLICKHOUSE_PORT_TCP}" ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " -[ -n "${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}" ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " -[ -n "${CLICKHOUSE_DATABASE}" ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " +[ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " +[ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " +[ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " +[ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " +[ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} [ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} [ -x "$CLICKHOUSE_BINARY" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY client} export CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} -export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT0} ${CLICKHOUSE_CLIENT_OPT}"} +export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT0:-} ${CLICKHOUSE_CLIENT_OPT:-}"} [ -x "${CLICKHOUSE_BINARY}-local" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY} local"} export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} @@ -42,7 +44,7 @@ export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:="8443"} export CLICKHOUSE_PORT_HTTP_PROTO=${CLICKHOUSE_PORT_HTTP_PROTO:="http"} # Add database to url params -if [ -n "${CLICKHOUSE_URL_PARAMS}" ] +if [ -v CLICKHOUSE_URL_PARAMS ] then export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&database=${CLICKHOUSE_DATABASE}" else @@ -53,7 +55,7 @@ export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICK export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"} # Add url params to url -if [ -n "${CLICKHOUSE_URL_PARAMS}" ] +if [ -v CLICKHOUSE_URL_PARAMS ] then export CLICKHOUSE_URL="${CLICKHOUSE_URL}?${CLICKHOUSE_URL_PARAMS}" export CLICKHOUSE_URL_HTTPS="${CLICKHOUSE_URL_HTTPS}?${CLICKHOUSE_URL_PARAMS}" @@ -65,7 +67,7 @@ export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PO export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="10"} -export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} +export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q -s --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py index e50ac0a3f8b..06610f6e42b 100755 --- a/tests/testflows/aes_encryption/regression.py +++ b/tests/testflows/aes_encryption/regression.py @@ -48,12 +48,13 @@ xfails = { @TestFeature @Name("aes encryption") @ArgumentParser(argparser) +@Specifications(SRS_008_ClickHouse_AES_Encryption_Functions) @Requirements( RQ_SRS008_AES_Functions("1.0"), RQ_SRS008_AES_Functions_DifferentModes("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse AES encryption functions regression module. """ nodes = { diff --git a/tests/testflows/aes_encryption/requirements/requirements.py b/tests/testflows/aes_encryption/requirements/requirements.py index bae8b5cc3c1..d92e159547d 100644 --- a/tests/testflows/aes_encryption/requirements/requirements.py +++ b/tests/testflows/aes_encryption/requirements/requirements.py @@ -1,10 +1,1956 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200731.1222107. +# document by TestFlows v1.6.201026.1232822. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_008_ClickHouse_AES_Encryption_Functions = Specification( + name='SRS-008 ClickHouse AES Encryption Functions', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-008 ClickHouse AES Encryption Functions +# Software Requirements Specification + +## Table of Contents +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS008.AES.Functions](#rqsrs008aesfunctions) + * 4.1.2 [RQ.SRS008.AES.Functions.Compatability.MySQL](#rqsrs008aesfunctionscompatabilitymysql) + * 4.1.3 [RQ.SRS008.AES.Functions.Compatability.Dictionaries](#rqsrs008aesfunctionscompatabilitydictionaries) + * 4.1.4 [RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL](#rqsrs008aesfunctionscompatabilityenginedatabasemysql) + * 4.1.5 [RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL](#rqsrs008aesfunctionscompatabilityenginetablemysql) + * 4.1.6 [RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL](#rqsrs008aesfunctionscompatabilitytablefunctionmysql) + * 4.1.7 [RQ.SRS008.AES.Functions.DifferentModes](#rqsrs008aesfunctionsdifferentmodes) + * 4.1.8 [RQ.SRS008.AES.Functions.DataFromMultipleSources](#rqsrs008aesfunctionsdatafrommultiplesources) + * 4.1.9 [RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues](#rqsrs008aesfunctionssuppressoutputofsensitivevalues) + * 4.1.10 [RQ.SRS008.AES.Functions.InvalidParameters](#rqsrs008aesfunctionsinvalidparameters) + * 4.1.11 [RQ.SRS008.AES.Functions.MismatchedKey](#rqsrs008aesfunctionsmismatchedkey) + * 4.1.12 [RQ.SRS008.AES.Functions.Check.Performance](#rqsrs008aesfunctionscheckperformance) + * 4.1.13 [RQ.SRS008.AES.Function.Check.Performance.BestCase](#rqsrs008aesfunctioncheckperformancebestcase) + * 4.1.14 [RQ.SRS008.AES.Function.Check.Performance.WorstCase](#rqsrs008aesfunctioncheckperformanceworstcase) + * 4.1.15 [RQ.SRS008.AES.Functions.Check.Compression](#rqsrs008aesfunctionscheckcompression) + * 4.1.16 [RQ.SRS008.AES.Functions.Check.Compression.LowCardinality](#rqsrs008aesfunctionscheckcompressionlowcardinality) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS008.AES.Encrypt.Function](#rqsrs008aesencryptfunction) + * 4.2.2 [RQ.SRS008.AES.Encrypt.Function.Syntax](#rqsrs008aesencryptfunctionsyntax) + * 4.2.3 [RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors](#rqsrs008aesencryptfunctionnisttestvectors) + * 4.2.4 [RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText](#rqsrs008aesencryptfunctionparametersplaintext) + * 4.2.5 [RQ.SRS008.AES.Encrypt.Function.Parameters.Key](#rqsrs008aesencryptfunctionparameterskey) + * 4.2.6 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode](#rqsrs008aesencryptfunctionparametersmode) + * 4.2.7 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesencryptfunctionparametersmodevaluesformat) + * 4.2.8 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesencryptfunctionparametersmodevalueinvalid) + * 4.2.9 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.10 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.11 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.12 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.13 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.14 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.15 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.16 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.17 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.18 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.19 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.20 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.21 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.22 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.23 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.24 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.25 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.26 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.27 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.28 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.29 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.30 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.31 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.32 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.33 [RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesencryptfunctionparametersinitializationvector) + * 4.2.34 [RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesencryptfunctionparametersadditionalauthenticateddata) + * 4.2.35 [RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesencryptfunctionparametersreturnvalue) + * 4.2.36 [RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesencryptfunctionkeylengthinvalidlengtherror) + * 4.2.37 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesencryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.38 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesencryptfunctioninitializationvectornotvalidformode) + * 4.2.39 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesencryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.40 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesencryptfunctionadditionalauthenticationdatalength) + * 4.2.41 [RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.42 [RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.43 [RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.44 [RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.45 [RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.46 [RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.47 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.48 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.49 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.50 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.51 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.52 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.53 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.54 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.55 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.56 [RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.57 [RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.58 [RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.59 [RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.60 [RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.61 [RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.62 [RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.63 [RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.64 [RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.2.65 [RQ.SRS008.AES.Decrypt.Function](#rqsrs008aesdecryptfunction) + * 4.2.66 [RQ.SRS008.AES.Decrypt.Function.Syntax](#rqsrs008aesdecryptfunctionsyntax) + * 4.2.67 [RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText](#rqsrs008aesdecryptfunctionparametersciphertext) + * 4.2.68 [RQ.SRS008.AES.Decrypt.Function.Parameters.Key](#rqsrs008aesdecryptfunctionparameterskey) + * 4.2.69 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode](#rqsrs008aesdecryptfunctionparametersmode) + * 4.2.70 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesdecryptfunctionparametersmodevaluesformat) + * 4.2.71 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesdecryptfunctionparametersmodevalueinvalid) + * 4.2.72 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.73 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.74 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.75 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.76 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.77 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.78 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.79 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.80 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.81 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.82 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.83 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.84 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.85 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.86 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.87 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.88 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.89 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.90 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.91 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.92 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.93 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.94 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.95 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.96 [RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesdecryptfunctionparametersinitializationvector) + * 4.2.97 [RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesdecryptfunctionparametersadditionalauthenticateddata) + * 4.2.98 [RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesdecryptfunctionparametersreturnvalue) + * 4.2.99 [RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesdecryptfunctionkeylengthinvalidlengtherror) + * 4.2.100 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesdecryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.101 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesdecryptfunctioninitializationvectornotvalidformode) + * 4.2.102 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesdecryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.103 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesdecryptfunctionadditionalauthenticationdatalength) + * 4.2.104 [RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.105 [RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.106 [RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.107 [RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.108 [RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.109 [RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.110 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.111 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.112 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.113 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.114 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.115 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.116 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.117 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.118 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.119 [RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.120 [RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.121 [RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.122 [RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.123 [RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.124 [RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.125 [RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.126 [RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.127 [RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.3 [MySQL Specific Functions](#mysql-specific-functions) + * 4.3.1 [RQ.SRS008.AES.MySQL.Encrypt.Function](#rqsrs008aesmysqlencryptfunction) + * 4.3.2 [RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax](#rqsrs008aesmysqlencryptfunctionsyntax) + * 4.3.3 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText](#rqsrs008aesmysqlencryptfunctionparametersplaintext) + * 4.3.4 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key](#rqsrs008aesmysqlencryptfunctionparameterskey) + * 4.3.5 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode](#rqsrs008aesmysqlencryptfunctionparametersmode) + * 4.3.6 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesformat) + * 4.3.7 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqlencryptfunctionparametersmodevalueinvalid) + * 4.3.8 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.9 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.10 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.11 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.12 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.13 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.14 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.15 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.16 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.17 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.18 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.19 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.20 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.21 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.22 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.23 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.24 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.25 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.26 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.27 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.28 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.29 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.30 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.31 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.32 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqlencryptfunctionparametersinitializationvector) + * 4.3.33 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqlencryptfunctionparametersreturnvalue) + * 4.3.34 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqlencryptfunctionkeylengthtooshorterror) + * 4.3.35 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqlencryptfunctionkeylengthtoolong) + * 4.3.36 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.37 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtoolong) + * 4.3.38 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqlencryptfunctioninitializationvectornotvalidformode) + * 4.3.39 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.40 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.41 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.42 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.43 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.44 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.45 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.46 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.47 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.48 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.49 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.50 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.51 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.52 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.53 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.54 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.55 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.56 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.3.57 [RQ.SRS008.AES.MySQL.Decrypt.Function](#rqsrs008aesmysqldecryptfunction) + * 4.3.58 [RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax](#rqsrs008aesmysqldecryptfunctionsyntax) + * 4.3.59 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText](#rqsrs008aesmysqldecryptfunctionparametersciphertext) + * 4.3.60 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key](#rqsrs008aesmysqldecryptfunctionparameterskey) + * 4.3.61 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode](#rqsrs008aesmysqldecryptfunctionparametersmode) + * 4.3.62 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesformat) + * 4.3.63 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqldecryptfunctionparametersmodevalueinvalid) + * 4.3.64 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.65 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.66 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.67 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.68 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.69 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.70 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.71 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.72 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.73 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.74 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.75 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.76 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.77 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.78 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.79 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.80 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.81 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.82 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.83 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.84 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.85 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.86 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.87 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.88 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqldecryptfunctionparametersinitializationvector) + * 4.3.89 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqldecryptfunctionparametersreturnvalue) + * 4.3.90 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqldecryptfunctionkeylengthtooshorterror) + * 4.3.91 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqldecryptfunctionkeylengthtoolong) + * 4.3.92 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.93 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtoolong) + * 4.3.94 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqldecryptfunctioninitializationvectornotvalidformode) + * 4.3.95 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.96 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.97 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.98 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.99 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.100 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.101 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.102 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.103 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.104 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.105 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.106 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.107 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.108 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.109 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.110 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.111 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.112 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ofbkeyandinitializationvectorlength) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +Users need an ability to encrypt and decrypt column data with tenant specific keys. +Use cases include protection of sensitive column values and [GDPR] right to forget policies. +The implementation will support capabilities of the [MySQL aes_encrypt] and [MySQL aes_decrypt] +functions which encrypt and decrypt values using the [AES] (Advanced Encryption Standard) +algorithm. This functionality will enable encryption and decryption of data +accessed on remote [MySQL] servers via [MySQL Dictionary] or [MySQL Database Engine], +[MySQL Table Engine], or [MySQL Table Function]. + +## Terminology + +* **AES** - + Advanced Encryption Standard ([AES]) + +## Requirements + +### Generic + +#### RQ.SRS008.AES.Functions +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data. + +#### RQ.SRS008.AES.Functions.Compatability.MySQL +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7]. + +#### RQ.SRS008.AES.Functions.Compatability.Dictionaries +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed on remote +[MySQL] servers using [MySQL Dictionary]. + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine], + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine]. + +#### RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function]. + +#### RQ.SRS008.AES.Functions.DifferentModes +version: 1.0 + +[ClickHouse] SHALL allow different modes to be supported in a single SQL statement +using explicit function parameters. + +#### RQ.SRS008.AES.Functions.DataFromMultipleSources +version: 1.0 + +[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources +in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary], +[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function] +with possibly different encryption schemes. + +#### RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues +version: 1.0 + +[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log, +error log, and `query_log` table to prevent leakage of sensitive values. + +#### RQ.SRS008.AES.Functions.InvalidParameters +version: 1.0 + +[ClickHouse] SHALL return an error when parameters are invalid. + +#### RQ.SRS008.AES.Functions.Mismatched.Key +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched keys. + +#### RQ.SRS008.AES.Functions.Mismatched.IV +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.AAD +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.Mode +version: 1.0 + +[ClickHouse] SHALL return an error or garbage for mismatched mode. + +#### RQ.SRS008.AES.Functions.Check.Performance +version: 1.0 + +Performance of [AES] encryption functions SHALL be measured. + +#### RQ.SRS008.AES.Function.Check.Performance.BestCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the best case +scenario where there is one key, one initialization vector, and one large stream of data. + +#### RQ.SRS008.AES.Function.Check.Performance.WorstCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the worst case +where there are `N` keys, `N` initialization vectors and `N` very small streams of data. + +#### RQ.SRS008.AES.Functions.Check.Compression +version: 1.0 + +Effect of [AES] encryption on column compression SHALL be measured. + +#### RQ.SRS008.AES.Functions.Check.Compression.LowCardinality +version: 1.0 + +Effect of [AES] encryption on the compression of a column with [LowCardinality] data type +SHALL be measured. + +### Specific + +#### RQ.SRS008.AES.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt` function + +```sql +aes_encrypt(plaintext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors +version: 1.0 + +[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt` function + +```sql +aes_decrypt(ciphertext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as +the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +### MySQL Specific Functions + +#### RQ.SRS008.AES.MySQL.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function + +```sql +aes_encrypt_mysql(plaintext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_encrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function + +```sql +aes_decrypt_mysql(ciphertext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting any data type as +the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_decrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +## References + +* **GDPR:** https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +* **MySQL:** https://www.mysql.com/ +* **AES:** https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +* **ClickHouse:** https://clickhouse.tech +* **Git:** https://git-scm.com/ + +[OpenSSL]: https://www.openssl.org/ +[LowCardinality]: https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ +[MergeTree]: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/ +[MySQL Database Engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/ +[MySQL Table Engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/ +[MySQL Table Function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/ +[MySQL Dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql +[GCM]: https://en.wikipedia.org/wiki/Galois/Counter_Mode +[CTR]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR) +[CBC]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_block_chaining_(CBC) +[ECB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_(ECB) +[CFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB1]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB8]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB128]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[OFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Output_feedback_(OFB) +[GDPR]: https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +[RFC5116]: https://tools.ietf.org/html/rfc5116#section-5.1 +[MySQL]: https://www.mysql.com/ +[MySQL 5.7]: https://dev.mysql.com/doc/refman/5.7/en/ +[MySQL aes_encrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-encrypt +[MySQL aes_decrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-decrypt +[AES]: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +[ClickHouse]: https://clickhouse.tech +[GitHub repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/aes_encryption/requirements/requirements.md +[Revision history]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/aes_encryption/requirements/requirements.md +[Git]: https://git-scm.com/ +[NIST test vectors]: https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program +''') + RQ_SRS008_AES_Functions = Requirement( name='RQ.SRS008.AES.Functions', version='1.0', @@ -14,9 +1960,9 @@ RQ_SRS008_AES_Functions = Requirement( uid=None, description=( '[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.MySQL', @@ -27,9 +1973,9 @@ RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Dictionaries', @@ -41,9 +1987,9 @@ RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( description=( '[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n' '[MySQL] servers using [MySQL Dictionary].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL', @@ -54,9 +2000,9 @@ RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL', @@ -67,9 +2013,9 @@ RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL', @@ -80,9 +2026,9 @@ RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_DifferentModes = Requirement( name='RQ.SRS008.AES.Functions.DifferentModes', @@ -94,9 +2040,9 @@ RQ_SRS008_AES_Functions_DifferentModes = Requirement( description=( '[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n' 'using explicit function parameters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( name='RQ.SRS008.AES.Functions.DataFromMultipleSources', @@ -110,9 +2056,9 @@ RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( 'in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n' '[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n' 'with possibly different encryption schemes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', @@ -124,9 +2070,9 @@ RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( description=( '[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n' 'error log, and `query_log` table to prevent leakage of sensitive values.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_InvalidParameters = Requirement( name='RQ.SRS008.AES.Functions.InvalidParameters', @@ -137,9 +2083,9 @@ RQ_SRS008_AES_Functions_InvalidParameters = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error when parameters are invalid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.Key', @@ -150,9 +2096,9 @@ RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched keys.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.IV', @@ -163,9 +2109,9 @@ RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.AAD', @@ -176,9 +2122,9 @@ RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.Mode', @@ -189,9 +2135,9 @@ RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error or garbage for mismatched mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Performance = Requirement( name='RQ.SRS008.AES.Functions.Check.Performance', @@ -202,9 +2148,9 @@ RQ_SRS008_AES_Functions_Check_Performance = Requirement( uid=None, description=( 'Performance of [AES] encryption functions SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( name='RQ.SRS008.AES.Function.Check.Performance.BestCase', @@ -216,9 +2162,9 @@ RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( description=( 'Performance of [AES] encryption functions SHALL be checked for the best case\n' 'scenario where there is one key, one initialization vector, and one large stream of data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', @@ -230,9 +2176,9 @@ RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( description=( 'Performance of [AES] encryption functions SHALL be checked for the worst case\n' 'where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Compression = Requirement( name='RQ.SRS008.AES.Functions.Check.Compression', @@ -243,9 +2189,9 @@ RQ_SRS008_AES_Functions_Check_Compression = Requirement( uid=None, description=( 'Effect of [AES] encryption on column compression SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', @@ -257,9 +2203,9 @@ RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( description=( 'Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n' 'SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function = Requirement( name='RQ.SRS008.AES.Encrypt.Function', @@ -270,9 +2216,9 @@ RQ_SRS008_AES_Encrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Syntax', @@ -287,9 +2233,9 @@ RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( '```sql\n' 'aes_encrypt(plaintext, key, mode, [iv, aad])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', @@ -300,9 +2246,9 @@ RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( uid=None, description=( '[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', @@ -314,9 +2260,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( description=( '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' 'the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', @@ -328,9 +2274,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', @@ -342,9 +2288,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', @@ -360,9 +2306,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', @@ -375,9 +2321,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -389,9 +2335,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -403,9 +2349,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -417,9 +2363,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -431,9 +2377,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -445,9 +2391,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -459,9 +2405,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -473,9 +2419,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -487,9 +2433,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -501,9 +2447,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -515,9 +2461,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -529,9 +2475,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -543,9 +2489,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -557,9 +2503,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -571,9 +2517,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -585,9 +2531,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -599,9 +2545,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -613,9 +2559,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -627,9 +2573,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM', @@ -643,9 +2589,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM', @@ -659,9 +2605,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM', @@ -675,9 +2621,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR', @@ -689,9 +2635,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR', @@ -703,9 +2649,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR', @@ -717,9 +2663,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', @@ -732,9 +2678,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', @@ -747,9 +2693,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirem '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' 'parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data\n' 'for block modes that require it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', @@ -761,9 +2707,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the encrypted value of the data\n' 'using `String` data type as the result of `aes_encrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', @@ -774,9 +2720,9 @@ RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', @@ -787,9 +2733,9 @@ RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', @@ -800,9 +2746,9 @@ RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requiremen uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', @@ -813,9 +2759,9 @@ RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Re uid=None, description=( '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', @@ -826,9 +2772,9 @@ RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement uid=None, description=( '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -840,9 +2786,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -854,9 +2800,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -868,9 +2814,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -882,9 +2828,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -896,9 +2842,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -910,9 +2856,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -924,9 +2870,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -938,9 +2884,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -952,9 +2898,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -966,9 +2912,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' 'and if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -980,9 +2926,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -994,9 +2940,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -1008,9 +2954,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -1022,9 +2968,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -1036,9 +2982,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -1050,9 +2996,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -1064,9 +3010,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -1078,9 +3024,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', @@ -1092,9 +3038,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', @@ -1106,9 +3052,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', @@ -1120,9 +3066,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', @@ -1134,9 +3080,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', @@ -1148,9 +3094,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', @@ -1162,9 +3108,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function = Requirement( name='RQ.SRS008.AES.Decrypt.Function', @@ -1175,9 +3121,9 @@ RQ_SRS008_AES_Decrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Syntax', @@ -1192,9 +3138,9 @@ RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( '```sql\n' 'aes_decrypt(ciphertext, key, mode, [iv, aad])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', @@ -1206,9 +3152,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( description=( '[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n' 'the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', @@ -1220,9 +3166,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', @@ -1234,9 +3180,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', @@ -1252,9 +3198,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', @@ -1267,9 +3213,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -1281,9 +3227,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -1295,9 +3241,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -1309,9 +3255,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -1323,9 +3269,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -1337,9 +3283,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -1351,9 +3297,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -1365,9 +3311,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -1379,9 +3325,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -1393,9 +3339,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -1407,9 +3353,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -1421,9 +3367,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -1435,9 +3381,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -1449,9 +3395,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -1463,9 +3409,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -1477,9 +3423,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -1491,9 +3437,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -1505,9 +3451,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -1519,9 +3465,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM', @@ -1535,9 +3481,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM', @@ -1551,9 +3497,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM', @@ -1567,9 +3513,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR', @@ -1581,9 +3527,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR', @@ -1595,9 +3541,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR', @@ -1609,9 +3555,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', @@ -1624,9 +3570,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', @@ -1639,9 +3585,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirem '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' 'parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data\n' 'for block modes that require it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', @@ -1653,9 +3599,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the decrypted value of the data\n' 'using `String` data type as the result of `aes_decrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', @@ -1666,9 +3612,9 @@ RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', @@ -1679,9 +3625,9 @@ RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', @@ -1693,9 +3639,9 @@ RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requiremen description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function\n' 'for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', @@ -1707,9 +3653,9 @@ RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Re description=( '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function\n' 'for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', @@ -1720,9 +3666,9 @@ RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement uid=None, description=( '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -1734,9 +3680,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -1748,9 +3694,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -1762,9 +3708,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -1776,9 +3722,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -1790,9 +3736,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -1804,9 +3750,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -1818,9 +3764,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -1832,9 +3778,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -1846,9 +3792,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -1860,9 +3806,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' 'and if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -1874,9 +3820,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' 'or `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -1888,9 +3834,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -1902,9 +3848,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -1916,9 +3862,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -1930,9 +3876,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -1944,9 +3890,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -1958,9 +3904,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -1972,9 +3918,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', @@ -1986,9 +3932,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', @@ -2000,9 +3946,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', @@ -2014,9 +3960,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', @@ -2028,9 +3974,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', @@ -2042,9 +3988,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', @@ -2056,9 +4002,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function', @@ -2069,9 +4015,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', @@ -2086,9 +4032,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( '```sql\n' 'aes_encrypt_mysql(plaintext, key, mode, [iv])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', @@ -2100,9 +4046,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( description=( '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' 'the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', @@ -2114,9 +4060,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', @@ -2128,9 +4074,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', @@ -2145,9 +4091,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', @@ -2160,9 +4106,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -2174,9 +4120,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Require description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -2188,9 +4134,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Require description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -2202,9 +4148,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Require description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -2216,9 +4162,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Require description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -2230,9 +4176,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Require description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -2244,9 +4190,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Require description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -2258,9 +4204,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -2272,9 +4218,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -2286,9 +4232,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -2300,9 +4246,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -2314,9 +4260,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -2328,9 +4274,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -2342,9 +4288,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -2356,9 +4302,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -2370,9 +4316,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -2384,9 +4330,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Require description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -2398,9 +4344,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Require description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -2412,9 +4358,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Require description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', @@ -2426,9 +4372,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', @@ -2440,9 +4386,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', @@ -2454,9 +4400,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', @@ -2468,9 +4414,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', @@ -2482,9 +4428,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', @@ -2496,9 +4442,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', @@ -2511,9 +4457,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requireme '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', @@ -2525,9 +4471,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the encrypted value of the data\n' 'using `String` data type as the result of `aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', @@ -2539,9 +4485,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( description=( '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n' 'function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', @@ -2563,9 +4509,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' '\t return folded_key\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', @@ -2577,9 +4523,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' 'that is required for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', @@ -2591,9 +4537,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requi description=( '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' 'its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', @@ -2605,9 +4551,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requ description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n' 'function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -2619,9 +4565,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -2633,9 +4579,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -2647,9 +4593,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -2661,9 +4607,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -2675,9 +4621,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -2689,9 +4635,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -2703,9 +4649,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -2717,9 +4663,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -2731,9 +4677,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -2745,9 +4691,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' 'and if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -2759,9 +4705,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -2773,9 +4719,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -2787,9 +4733,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -2801,9 +4747,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -2815,9 +4761,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -2829,9 +4775,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -2843,9 +4789,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -2857,9 +4803,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function', @@ -2870,9 +4816,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', @@ -2887,9 +4833,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( '```sql\n' 'aes_decrypt_mysql(ciphertext, key, mode, [iv])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', @@ -2901,9 +4847,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( description=( '[ClickHouse] SHALL support `ciphertext` accepting any data type as\n' 'the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', @@ -2915,9 +4861,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', @@ -2929,9 +4875,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', @@ -2946,9 +4892,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', @@ -2961,9 +4907,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -2975,9 +4921,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Require description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -2989,9 +4935,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Require description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -3003,9 +4949,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Require description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -3017,9 +4963,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Require description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -3031,9 +4977,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Require description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -3045,9 +4991,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Require description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -3059,9 +5005,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -3073,9 +5019,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -3087,9 +5033,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -3101,9 +5047,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -3115,9 +5061,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -3129,9 +5075,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -3143,9 +5089,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -3157,9 +5103,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -3171,9 +5117,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -3185,9 +5131,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Require description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -3199,9 +5145,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Require description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -3213,9 +5159,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Require description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', @@ -3227,9 +5173,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', @@ -3241,9 +5187,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', @@ -3255,9 +5201,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', @@ -3269,9 +5215,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', @@ -3283,9 +5229,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', @@ -3297,9 +5243,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', @@ -3312,9 +5258,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requireme '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', @@ -3326,9 +5272,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the decrypted value of the data\n' 'using `String` data type as the result of `aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', @@ -3340,9 +5286,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( description=( '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n' 'function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', @@ -3364,9 +5310,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' '\t return folded_key\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', @@ -3378,9 +5324,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' 'that is required for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', @@ -3392,9 +5338,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requi description=( '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' 'its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', @@ -3406,9 +5352,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requ description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n' 'function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -3420,9 +5366,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -3434,9 +5380,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -3448,9 +5394,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -3462,9 +5408,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -3476,9 +5422,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -3490,9 +5436,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -3504,9 +5450,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -3518,9 +5464,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -3532,9 +5478,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -3546,9 +5492,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' 'and if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -3560,9 +5506,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -3574,9 +5520,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -3588,9 +5534,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -3602,9 +5548,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -3616,9 +5562,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -3630,9 +5576,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -3644,9 +5590,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -3658,6 +5604,6 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) diff --git a/tests/testflows/example/regression.py b/tests/testflows/example/regression.py index 2c0a778d39b..cb58b42ba4c 100755 --- a/tests/testflows/example/regression.py +++ b/tests/testflows/example/regression.py @@ -2,7 +2,7 @@ import sys from testflows.core import * -append_path(sys.path, "..") +append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser @@ -10,13 +10,13 @@ from helpers.argparser import argparser @TestFeature @Name("example") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """Simple example of how you can use TestFlows to test ClickHouse. """ nodes = { "clickhouse": ("clickhouse1",), } - + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster diff --git a/tests/testflows/helpers/argparser.py b/tests/testflows/helpers/argparser.py index 033c15a3bfe..03014becb76 100644 --- a/tests/testflows/helpers/argparser.py +++ b/tests/testflows/helpers/argparser.py @@ -1,5 +1,12 @@ import os +def onoff(v): + if v in ["yes", "1", "on"]: + return True + elif v in ["no", "0", "off"]: + return False + raise ValueError(f"invalid {v}") + def argparser(parser): """Default argument parser for regressions. """ @@ -10,4 +17,10 @@ def argparser(parser): parser.add_argument("--clickhouse-binary-path", type=str, dest="clickhouse_binary_path", help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path", - default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) \ No newline at end of file + default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) + + parser.add_argument("--stress", action="store_true", default=False, + help="enable stress testing (might take a long time)") + + parser.add_argument("--parallel", type=onoff, default=True, choices=["yes", "no", "on", "off", 0, 1], + help="enable parallelism for tests that support it") \ No newline at end of file diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py old mode 100644 new mode 100755 index 01ebfa8424b..d173547a916 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -7,6 +7,7 @@ import tempfile from testflows.core import * from testflows.asserts import error from testflows.connect import Shell +from testflows.uexpect import ExpectTimeoutError class QueryRuntimeException(Exception): """Exception during query execution on the server. @@ -25,7 +26,7 @@ class Node(object): def repr(self): return f"Node(name='{self.name}')" - def restart(self, timeout=120, safe=True): + def restart(self, timeout=300, safe=True): """Restart node. """ with self.cluster.lock: @@ -42,18 +43,46 @@ class Node(object): class ClickHouseNode(Node): """Node with ClickHouse server. """ - def wait_healthy(self, timeout=120): + def wait_healthy(self, timeout=300): with By(f"waiting until container {self.name} is healthy"): start_time = time.time() while True: - if self.query("select 1", no_checks=1, timeout=120, steps=False).exitcode == 0: + if self.query("select 1", no_checks=1, timeout=300, steps=False).exitcode == 0: break if time.time() - start_time < timeout: time.sleep(2) continue assert False, "container is not healthy" - def restart(self, timeout=120, safe=True, wait_healthy=True): + def stop(self, timeout=300, safe=True): + """Stop node. + """ + if safe: + self.query("SYSTEM STOP MOVES") + self.query("SYSTEM STOP MERGES") + self.query("SYSTEM FLUSH LOGS") + with By("waiting for 5 sec for moves and merges to stop"): + time.sleep(5) + with And("forcing to sync everything to disk"): + self.command("sync", timeout=30) + + with self.cluster.lock: + for key in list(self.cluster._bash.keys()): + if key.endswith(f"-{self.name}"): + shell = self.cluster._bash.pop(key) + shell.__exit__(None, None, None) + + self.cluster.command(None, f'{self.cluster.docker_compose} stop {self.name}', timeout=timeout) + + def start(self, timeout=300, wait_healthy=True): + """Start node. + """ + self.cluster.command(None, f'{self.cluster.docker_compose} start {self.name}', timeout=timeout) + + if wait_healthy: + self.wait_healthy(timeout) + + def restart(self, timeout=300, safe=True, wait_healthy=True): """Restart node. """ if safe: @@ -79,32 +108,43 @@ class ClickHouseNode(Node): def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, raise_on_exception=False, step=By, settings=None, *args, **kwargs): """Execute and check query. - :param sql: sql query :param message: expected message that should be in the output, default: None :param exitcode: expected exitcode, default: None """ + settings = list(settings or []) + + if hasattr(current().context, "default_query_settings"): + settings += current().context.default_query_settings + if len(sql) > 1024: with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: query.write(sql) query.flush() command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n" - for setting in settings or []: + for setting in settings: name, value = setting command += f" --{name} \"{value}\"" description = f""" echo -e \"{sql[:100]}...\" > {query.name} {command} """ - with step("executing command", description=description) if steps else NullStep(): - r = self.cluster.bash(None)(command, *args, **kwargs) + with step("executing command", description=description, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(None)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(None) else: command = f"echo -e \"{sql}\" | clickhouse client -n" - for setting in settings or []: + for setting in settings: name, value = setting command += f" --{name} \"{value}\"" - with step("executing command", description=command) if steps else NullStep(): - r = self.cluster.bash(self.name)(command, *args, **kwargs) + with step("executing command", description=command, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(self.name)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(self.name) + raise if no_checks: return r @@ -135,6 +175,7 @@ class Cluster(object): docker_compose="docker-compose", docker_compose_project_dir=None, docker_compose_file="docker-compose.yml"): + self.terminating = False self._bash = {} self.clickhouse_binary_path = clickhouse_binary_path self.configs_dir = configs_dir @@ -168,7 +209,7 @@ class Cluster(object): self.docker_compose += f" --no-ansi --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" self.lock = threading.Lock() - def shell(self, node, timeout=120): + def shell(self, node, timeout=300): """Returns unique shell terminal to be used. """ if node is None: @@ -181,14 +222,22 @@ class Cluster(object): shell.timeout = timeout return shell - def bash(self, node, timeout=120): + def bash(self, node, timeout=300): """Returns thread-local bash terminal to a specific node. - :param node: name of the service """ + test = current() + + if self.terminating: + if test and (test.cflags & MANDATORY): + pass + else: + raise InterruptedError("terminating") + current_thread = threading.current_thread() - id = f"{current_thread.ident}-{node}" + id = f"{current_thread.name}-{node}" + with self.lock: if self._bash.get(id) is None: if node is None: @@ -197,9 +246,30 @@ class Cluster(object): self._bash[id] = Shell(command=[ "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" ], name=node).__enter__() + self._bash[id].timeout = timeout + + # clean up any stale open shells for threads that have exited + active_thread_names = {thread.name for thread in threading.enumerate()} + + for bash_id in list(self._bash.keys()): + thread_name, node_name = bash_id.rsplit("-", 1) + if thread_name not in active_thread_names: + self._bash[bash_id].__exit__(None, None, None) + del self._bash[bash_id] + return self._bash[id] + def close_bash(self, node): + current_thread = threading.current_thread() + id = f"{current_thread.name}-{node}" + + with self.lock: + if self._bash.get(id) is None: + return + self._bash[id].__exit__(None, None, None) + del self._bash[id] + def __enter__(self): with Given("docker-compose cluster"): self.up() @@ -211,20 +281,21 @@ class Cluster(object): self.down() finally: with self.lock: - for shell in list(self._bash.values()): + for shell in self._bash.values(): shell.__exit__(type, value, traceback) def node(self, name): """Get object with node bound methods. - :param name: name of service name """ if name.startswith("clickhouse"): return ClickHouseNode(self, name) return Node(self, name) - def down(self, timeout=120): + def down(self, timeout=300): """Bring cluster down by executing docker-compose down.""" + self.terminating = True + try: bash = self.bash(None) with self.lock: @@ -236,7 +307,7 @@ class Cluster(object): else: self._bash[id] = shell finally: - return self.command(None, f"{self.docker_compose} down", timeout=timeout) + return self.command(None, f"{self.docker_compose} down --timeout 60", bash=bash, timeout=timeout) def up(self, timeout=30*60): if self.local: @@ -265,7 +336,7 @@ class Cluster(object): if cmd.exitcode != 0: continue with And("executing docker-compose down just in case it is up"): - cmd = self.command(None, f"{self.docker_compose} down 2>&1 | tee", exitcode=None, timeout=timeout) + cmd = self.command(None, f"{self.docker_compose} down --remove-orphans 2>&1 | tee", exitcode=None, timeout=timeout) if cmd.exitcode != 0: continue with And("executing docker-compose up"): @@ -286,18 +357,22 @@ class Cluster(object): for name in self.nodes["clickhouse"]: self.node(name).wait_healthy() - def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs): + def command(self, node, command, message=None, exitcode=None, steps=True, bash=None, *args, **kwargs): """Execute and check command. - :param node: name of the service :param command: command :param message: expected message that should be in the output, default: None :param exitcode: expected exitcode, default: None :param steps: don't break command into steps, default: True """ - debug(f"command() {node}, {command}") with By("executing command", description=command, format_description=False) if steps else NullStep(): - r = self.bash(node)(command, *args, **kwargs) + if bash is None: + bash = self.bash(node) + try: + r = bash(command, *args, **kwargs) + except ExpectTimeoutError: + self.close_bash(node) + raise if exitcode is not None: with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep(): assert r.exitcode == exitcode, error(r.output) diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index 9d0a5ca743f..ed75ce4fe75 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -33,7 +33,7 @@ xfails = { RQ_SRS_007_LDAP_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse integration with LDAP regression module. """ nodes = { @@ -43,6 +43,11 @@ def regression(self, local, clickhouse_binary_path): with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster + if stress is not None or not hasattr(self.context, "stress"): + self.context.stress = stress + if parallel is not None or not hasattr(self.context, "parallel"): + self.context.parallel = parallel + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.authentication.tests.multiple_servers", "scenario")) Feature(run=load("ldap.authentication.tests.connections", "feature")) diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index d322db70330..1c65a29fef4 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -460,14 +460,14 @@ time user configuration contains any of the `` entries. #### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined version: 1.0 -[ClickHouse] SHALL throw an error during any authentification attempt +[ClickHouse] SHALL throw an error during any authentication attempt if the name of the [LDAP] server used inside the `` entry is not defined in the `` section. #### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty version: 1.0 -[ClickHouse] SHALL throw an error during any authentification attempt +[ClickHouse] SHALL throw an error during any authentication attempt if the name of the [LDAP] server used inside the `` entry is empty. diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 967e097d1fa..60fbef9b8cd 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -1,10 +1,571 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200811.1124123. +# document by TestFlows v1.6.201026.1232822. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( + name='SRS-007 ClickHouse Authentication of Users via LDAP', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-007 ClickHouse Authentication of Users via LDAP + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) + * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) + * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) + * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) + * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) + * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) + * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) + * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) + * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) + * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) + * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) + * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) + * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) + * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) + * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) + * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) + * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) + * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) + * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) + * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) + * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) + * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) + * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) + * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) + * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) + * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) + * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) + * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) + * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) + * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) + * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) + * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) + * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) + * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) + * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) + * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) + * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) + * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) + * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) + * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) + * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) + * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) + * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) + * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) + * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) + * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) + * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) + * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.38 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.39 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +[ClickHouse] currently does not have any integration with [LDAP]. +As the initial step in integrating with [LDAP] this software requirements specification covers +only the requirements to enable authentication of users using an [LDAP] server. + +## Terminology + +* **CA** - + Certificate Authority ([CA]) + +* **LDAP** - + Lightweight Directory Access Protocol ([LDAP]) + +## Requirements + +### Generic + +#### RQ.SRS-007.LDAP.Authentication +version: 1.0 + +[ClickHouse] SHALL support user authentication via an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate +users. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. + +#### RQ.SRS-007.LDAP.Server.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +#### RQ.SRS-007.LDAP.User.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the user +has been deleted from the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications. + +### Specific + +#### RQ.SRS-007.LDAP.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. + +#### RQ.SRS-007.LDAP.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +#### RQ.SRS-007.LDAP.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.RBAC +version: 1.0 + +[ClickHouse] SHALL support creating users identified using an [LDAP] server using +the following RBAC command + +```sql +CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using +an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. + +```xml + + + + + my_ldap_server + + + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty +version: 1.0 + +[ClickHouse] SHALL not support empty string as a user name. + +#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP +version: 1.0 + +[ClickHouse] SHALL throw an error if `` is specified for the user and at the same +time user configuration contains any of the `` entries. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentication attempt +if the name of the [LDAP] server used inside the `` entry +is not defined in the `` section. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentication attempt +if the name of the [LDAP] server used inside the `` entry +is empty. + +#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer +version: 1.0 + +[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Long +version: 1.0 + +[ClickHouse] SHALL support long user names of at least 256 bytes +to specify users that can be authenticated using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 +version: 1.0 + +[ClickHouse] SHALL support user names that contain [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username. + +#### RQ.SRS-007.LDAP.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. + +#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users using an [LDAP] server. + +## References + +* **ClickHouse:** https://clickhouse.tech + +[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind +[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind +[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind +[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 +[OpenSSL]: https://www.openssl.org/ +[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html +[CA]: https://en.wikipedia.org/wiki/Certificate_authority +[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://github.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Git]: https://git-scm.com/ +''') + RQ_SRS_007_LDAP_Authentication = Requirement( name='RQ.SRS-007.LDAP.Authentication', version='1.0', @@ -14,9 +575,9 @@ RQ_SRS_007_LDAP_Authentication = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( name='RQ.SRS-007.LDAP.Authentication.MultipleServers', @@ -28,9 +589,9 @@ RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( description=( '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' 'users.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', @@ -41,9 +602,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', @@ -54,9 +615,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', @@ -68,9 +629,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( description=( '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' 'plain text `ldap://` protocol that is upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', @@ -81,9 +642,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( uid=None, description=( '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', @@ -94,9 +655,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( uid=None, description=( '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', @@ -107,9 +668,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = uid=None, description=( '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', @@ -120,9 +681,9 @@ RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.User.Configuration.Invalid', @@ -133,9 +694,9 @@ RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', @@ -147,9 +708,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', @@ -161,9 +722,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', @@ -175,9 +736,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( description=( '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Valid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Valid', @@ -189,9 +750,9 @@ RQ_SRS_007_LDAP_Authentication_Valid = Requirement( description=( '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' 'user name and password match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid', @@ -203,9 +764,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' 'do not match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', @@ -217,9 +778,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' 'has been deleted from the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', @@ -231,9 +792,9 @@ RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' 'on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', @@ -245,9 +806,9 @@ RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' 'for the user is changed on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', @@ -258,9 +819,9 @@ RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', @@ -271,9 +832,9 @@ RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel', @@ -284,9 +845,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( uid=None, description=( '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', @@ -299,9 +860,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( '[ClickHouse] SHALL support authentication of valid users and\n' 'prohibit authentication of invalid users using [LDAP] server\n' 'in parallel without having invalid attempts affecting valid authentications.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_UnreachableServer = Requirement( name='RQ.SRS-007.LDAP.UnreachableServer', @@ -312,9 +873,9 @@ RQ_SRS_007_LDAP_UnreachableServer = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Name', @@ -325,9 +886,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a server name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Host', @@ -339,9 +900,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port', @@ -352,9 +913,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', @@ -365,9 +926,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( uid=None, description=( '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', @@ -379,9 +940,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the prefix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', @@ -393,9 +954,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the suffix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', @@ -408,9 +969,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' '\n' "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', @@ -421,9 +982,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', @@ -435,9 +996,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( description=( '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' 'to enable SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', @@ -449,9 +1010,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( description=( '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' 'plain text `ldap://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', @@ -463,9 +1024,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( description=( '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' 'SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', @@ -477,9 +1038,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( description=( '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', @@ -491,9 +1052,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify\n' 'the minimum protocol version of SSL/TLS.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', @@ -505,9 +1066,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirem description=( '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' 'as a value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', @@ -518,9 +1079,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Require uid=None, description=( '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', @@ -532,9 +1093,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' 'certificate verification behavior.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', @@ -545,9 +1106,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requiremen uid=None, description=( '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', @@ -560,9 +1121,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' 'provided, the session SHALL be immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', @@ -576,9 +1137,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( 'enable requesting of client certificate. If no\n' 'certificate is provided, the session SHALL proceed normally.\n' 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', @@ -592,9 +1153,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( 'enable requesting of client certificate. If no certificate is provided, the session\n' 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' 'immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', @@ -606,9 +1167,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( description=( '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' 'disable requesting of client certificate.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', @@ -620,9 +1181,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' '[ClickHouse] to establish connection with the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', @@ -634,9 +1195,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' 'specified by the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', @@ -648,9 +1209,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify to a path to\n' 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', @@ -662,9 +1223,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', @@ -685,9 +1246,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( '\n' 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' '[ClickHouse] and therefore might change.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', @@ -718,9 +1279,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.RBAC', @@ -736,9 +1297,9 @@ RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( '```sql\n' "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Syntax', @@ -762,9 +1323,9 @@ RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', @@ -775,9 +1336,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a user name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', @@ -789,9 +1350,9 @@ RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( description=( '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' 'time user configuration contains any of the `` entries.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', @@ -801,12 +1362,12 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requireme type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' + '[ClickHouse] SHALL throw an error during any authentication attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is not defined in the `` section.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', @@ -816,12 +1377,12 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' + '[ClickHouse] SHALL throw an error during any authentication attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', @@ -832,9 +1393,9 @@ RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( uid=None, description=( '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', @@ -846,9 +1407,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( description=( '[ClickHouse] SHALL support long user names of at least 256 bytes\n' 'to specify users that can be authenticated using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', @@ -859,9 +1420,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Empty', @@ -872,9 +1433,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support authenticating users with empty username.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Long', @@ -885,9 +1446,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', @@ -898,9 +1459,9 @@ RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Empty', @@ -913,9 +1474,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( '[ClickHouse] SHALL not support authenticating users with empty passwords\n' 'even if an empty password is valid for the user and\n' 'is allowed by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Long', @@ -927,9 +1488,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( description=( '[ClickHouse] SHALL support long password of at least 256 bytes\n' 'that can be used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', @@ -941,6 +1502,6 @@ RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( description=( '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' 'used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 0fd9670fae0..ed8d46df92b 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -85,6 +85,8 @@ def add_config(config, timeout=60, restart=False): :param config: configuration file description :param timeout: timeout, default: 20 sec """ + node = current().context.node + def check_preprocessed_config_is_updated(after_removal=False): """Check that preprocessed config is updated. """ @@ -116,13 +118,16 @@ def add_config(config, timeout=60, restart=False): with When("I close terminal to the node to be restarted"): bash.close() - with And("I get the current log size"): - logsize = \ - node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ - 0].strip() + with And("I stop ClickHouse to apply the config changes"): + node.stop(safe=False) - with And("I restart ClickHouse to apply the config changes"): - node.restart(safe=False) + with And("I get the current log size"): + cmd = node.cluster.command(None, + f"stat --format=%s {os.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log") + logsize = cmd.output.split(" ")[0].strip() + + with And("I start ClickHouse back up"): + node.start() with Then("I tail the log file from using previous log size as the offset"): bash.prompt = bash.__class__.prompt @@ -139,7 +144,6 @@ def add_config(config, timeout=60, restart=False): f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) - node = current().context.node try: with Given(f"{config.name}"): if settings.debug: diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py index 6ce860a6fd2..bd404d54438 100755 --- a/tests/testflows/ldap/external_user_directory/regression.py +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -33,7 +33,7 @@ xfails = { RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse LDAP external user directory regression module. """ nodes = { @@ -42,6 +42,11 @@ def regression(self, local, clickhouse_binary_path): with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster + + if stress is not None or not hasattr(self.context, "stress"): + self.context.stress = stress + if parallel is not None or not hasattr(self.context, "parallel"): + self.context.parallel = parallel Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.external_user_directory.tests.simple", "scenario")) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 9b216e7dd30..47c10121b68 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -92,25 +92,23 @@ def parallel_login(self, server, user_count=10, timeout=200): with Given("a group of LDAP users"): users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*users): - tasks = [] - try: - with When("users try to login in parallel", description=""" - * with valid username and password - * with invalid username and valid password - * with valid username and invalid password - """): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + with ldap_users(*users): + tasks = [] + try: + with When("users try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( @@ -127,25 +125,23 @@ def parallel_login_with_the_same_user(self, server, timeout=200): with Given("only one LDAP user"): users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*users): - tasks = [] - try: - with When("the same user tries to login in parallel", description=""" - * with valid username and password - * with invalid username and valid password - * with valid username and invalid password - """): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + with ldap_users(*users): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario def login_after_ldap_external_user_directory_is_removed(self, server): @@ -162,6 +158,7 @@ def login_after_ldap_external_user_directory_is_removed(self, server): login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -204,6 +201,7 @@ def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=200 join(tasks, timeout) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -256,6 +254,7 @@ def parallel_login_with_multiple_servers(self, server, user_count=10, timeout=20 join(tasks, timeout) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -323,20 +322,18 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200): users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with rbac_users(*users): - tasks = [] - try: - with When("I login in parallel"): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + with rbac_users(*users): + tasks = [] + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( @@ -347,10 +344,8 @@ def login_after_user_is_added_to_ldap(self, server): """ user = {"cn": "myuser", "userpassword": "myuser"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with When(f"I add user to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server) + with When(f"I add user to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -363,27 +358,25 @@ def login_after_user_is_deleted_from_ldap(self, server): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I delete this user from LDAP"): - delete_user_from_ldap(user) + with When("I delete this user from LDAP"): + delete_user_from_ldap(user) - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -396,31 +389,29 @@ def login_after_user_password_changed_in_ldap(self, server): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") + with When("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) - with And("when I try to login with the new password it should work"): - login_and_execute_query(username=user["cn"], password="newpassword") + with And("when I try to login with the new password it should work"): + login_and_execute_query(username=user["cn"], password="newpassword") - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -434,27 +425,25 @@ def login_after_user_cn_changed_in_ldap(self, server): user = None new_user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I change user password in LDAP"): - new_user = change_user_cn_in_ldap(user, "myuser2") + with When("I change user password in LDAP"): + new_user = change_user_cn_in_ldap(user, "myuser2") - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if new_user is not None: - delete_user_from_ldap(new_user, exitcode=None) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) @TestScenario @Requirements( @@ -467,31 +456,29 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I restart LDAP server"): - self.context.ldap_node.restart() + with When("I restart LDAP server"): + self.context.ldap_node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -504,31 +491,29 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I restart ClickHouse server"): - self.context.node.restart() + with When("I restart ClickHouse server"): + self.context.node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -542,9 +527,7 @@ def valid_username_with_valid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -561,9 +544,7 @@ def valid_username_and_invalid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -575,10 +556,8 @@ def valid_username_and_password(self, server): username = "valid_username_and_password" user = {"cn": username, "userpassword": username} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with When(f"I add user {username} to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server) + with When(f"I add user {username} to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -593,9 +572,7 @@ def valid_username_and_password_invalid_server(self, server=None): exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) @TestScenario @Requirements( @@ -608,9 +585,7 @@ def valid_long_username_and_short_password(self, server): username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -626,9 +601,7 @@ def invalid_long_username_and_valid_short_password(self, server): exitcode = 4 message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -641,9 +614,7 @@ def valid_short_username_and_long_password(self, server): username = "long_password" user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -659,9 +630,7 @@ def valid_short_username_and_invalid_long_password(self, server): exitcode = 4 message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -677,9 +646,7 @@ def valid_username_and_invalid_password(self, server): exitcode = 4 message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -695,9 +662,7 @@ def invalid_username_and_valid_password(self, server): exitcode = 4 message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -710,9 +675,7 @@ def valid_utf8_username_and_ascii_password(self, server): username = "utf8_username_Gãńdåłf_Thê_Gręât" user = {"cn": username, "userpassword": "utf8_username"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -725,18 +688,14 @@ def valid_ascii_username_and_utf8_password(self, server): username = "utf8_password" user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario def empty_username_and_empty_password(self, server=None): """Check that we can login using empty username and empty password as it will use the default user and that has an empty password. """ - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - login_and_execute_query(username="", password="") + login_and_execute_query(username="", password="") @TestScenario @Requirements( @@ -763,18 +722,16 @@ def user_lookup_priority(self, server): "ldap": {"username": "ldap", "password": "userldap"} } - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): - with rbac_users({"cn": "local", "userpassword": "local"}): - with When("I try to login as 'default' user which is also defined in users.xml it should fail"): - login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): + with rbac_users({"cn": "local", "userpassword": "local"}): + with When("I try to login as 'default' user which is also defined in users.xml it should fail"): + login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) - with When("I try to login as 'local' user which is also defined in local storage it should fail"): - login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) + with When("I try to login as 'local' user which is also defined in local storage it should fail"): + login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) - with When("I try to login as 'ldap' user defined only in LDAP it should work"): - login_and_execute_query(**users["ldap"]) + with When("I try to login as 'ldap' user defined only in LDAP it should work"): + login_and_execute_query(**users["ldap"]) @TestOutline(Feature) @@ -795,5 +752,10 @@ def feature(self, servers=None, server=None, node="clickhouse1"): server = "openldap1" with ldap_servers(servers): - for scenario in loads(current_module(), Scenario): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + for scenario in loads(current_module(), Scenario, filter=~has.tag("custom config")): + Scenario(test=scenario, flags=TE)(server=server) + + for scenario in loads(current_module(), Scenario, filter=has.tag("custom config")): Scenario(test=scenario, flags=TE)(server=server) diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 0e9d06cf84a..9cc9aa85f93 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -9,7 +9,7 @@ from helpers.argparser import argparser @TestModule @Name("ldap") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, parallel=None, stress=None): """ClickHouse LDAP integration regression module. """ args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} @@ -18,4 +18,4 @@ def regression(self, local, clickhouse_binary_path): Feature(test=load("ldap.external_user_directory.regression", "regression"))(**args) if main(): - regression() + regression() \ No newline at end of file diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml index ada8eec5fc9..a7ed0d6e2b4 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml @@ -58,9 +58,44 @@ 9440 1 - - - + + + + + + clickhouse1 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + clickhouse2 @@ -73,8 +108,20 @@ 9000 - - + + + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + clickhouse1 @@ -94,6 +141,22 @@ + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml index ca65ffd5e04..768d2250b79 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml @@ -3,6 +3,7 @@ /etc/clickhouse-server/ssl/server.crt /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem none true diff --git a/tests/testflows/rbac/configs/clickhouse/config.xml b/tests/testflows/rbac/configs/clickhouse/config.xml index 65187edf806..4ec12232539 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.xml @@ -69,7 +69,7 @@ - + 0.0.0.0 /var/lib/clickhouse/access/ + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + users.xml @@ -160,7 +172,7 @@ - + @@ -220,7 +232,7 @@ See https://clickhouse.yandex/docs/en/table_engines/replication/ --> - + - +