diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e73d97c8123..7d410f833c5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -3105,10 +3105,10 @@ jobs: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread + TEMP_PATH=${{runner.temp}}/stress_asan REPORTS_PATH=${{runner.temp}}/reports_dir CHECK_NAME=Stress test (asan) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse + REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse EOF - name: Download json reports uses: actions/download-artifact@v3 @@ -3267,6 +3267,142 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" + ############################################################################################## + ######################################### UPGRADE CHECK ###################################### + ############################################################################################## + UpgradeCheckAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/upgrade_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Upgrade check (asan) + REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Upgrade check + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 upgrade_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + UpgradeCheckTsan: + needs: [BuilderDebTsan] + # same as for stress test with tsan + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/upgrade_thread + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Upgrade check (tsan) + REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Upgrade check + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 upgrade_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + UpgradeCheckMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/upgrade_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Upgrade check (msan) + REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Upgrade check + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 upgrade_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + UpgradeCheckDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/upgrade_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Upgrade check (debug) + REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Upgrade check + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 upgrade_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################## ##################################### AST FUZZERS ############################################ ############################################################################################## diff --git a/base/glibc-compatibility/glibc-compatibility.c b/base/glibc-compatibility/glibc-compatibility.c index bae03ad590a..7e8ea5051d7 100644 --- a/base/glibc-compatibility/glibc-compatibility.c +++ b/base/glibc-compatibility/glibc-compatibility.c @@ -195,7 +195,6 @@ long splice(int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, un #include #include -#if !defined(__aarch64__) struct statx { uint32_t stx_mask; uint32_t stx_blksize; @@ -226,7 +225,6 @@ int statx(int fd, const char *restrict path, int flag, { return syscall(SYS_statx, fd, path, flag, mask, statxbuf); } -#endif #include diff --git a/docker/images.json b/docker/images.json index 78e7729671e..508138d79af 100644 --- a/docker/images.json +++ b/docker/images.json @@ -43,7 +43,8 @@ "docker/test/stateful": { "name": "clickhouse/stateful-test", "dependent": [ - "docker/test/stress" + "docker/test/stress", + "docker/test/upgrade" ] }, "docker/test/unit": { @@ -54,6 +55,10 @@ "name": "clickhouse/stress-test", "dependent": [] }, + "docker/test/upgrade": { + "name": "clickhouse/upgrade-check", + "dependent": [] + }, "docker/test/codebrowser": { "name": "clickhouse/codebrowser", "dependent": [] diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 961c528f19c..5dbb244c298 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844 ARG DEBIAN_FRONTEND=noninteractive @@ -9,13 +9,14 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list && groupadd -r clickhouse --gid=101 \ && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \ && apt-get update \ + && apt-get upgrade -yq \ && apt-get install --yes --no-install-recommends \ apt-transport-https \ ca-certificates \ dirmngr \ - gnupg \ - locales \ + gnupg2 \ wget \ + locales \ tzdata \ && apt-get clean @@ -80,15 +81,8 @@ RUN arch=${TARGETARCH:-amd64} \ && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client -# Remove as much of Ubuntu as possible. -# ClickHouse does not need Ubuntu. It can run on top of Linux kernel without any OS distribution. -# ClickHouse does not need Docker at all. ClickHouse is above all that. -# It does not care about Ubuntu, Docker, or other cruft and you should neither. -# The fact that this Docker image is based on Ubuntu is just a misconception. -# Some vulnerability scanners are arguing about Ubuntu, which is not relevant to ClickHouse at all. -# ClickHouse does not care when you report false vulnerabilities by running some Docker scanners. - -RUN apt-get remove --purge -y libksba8 && apt-get autoremove -y +RUN apt-get autoremove --purge -yq libksba8 && \ + apt-get autoremove -yq # we need to allow "others" access to clickhouse folder, because docker container # can be started with arbitrary uid (openshift usecase) diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index b67a638188c..71a2e92e3a8 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,4 +1,4 @@ -# rebuild in #33610 +# rebuild in #47031 # docker build -t clickhouse/stateful-test . ARG FROM_TAG=latest FROM clickhouse/stateless-test:$FROM_TAG diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 393508fd551..e9712f430fd 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -21,10 +21,9 @@ RUN apt-get update -y \ openssl \ netcat-openbsd \ telnet \ - llvm-9 \ - brotli + brotli \ + && apt-get clean -COPY ./stress /stress COPY run.sh / ENV DATASETS="hits visits" diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 95704336468..15f58d6c3a3 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -8,229 +8,13 @@ dmesg --clear set -x -# core.COMM.PID-TID -sysctl kernel.core_pattern='core.%e.%p-%P' +# we mount tests folder from repo to /usr/share +ln -s /usr/share/clickhouse-test/ci/stress.py /usr/bin/stress +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test -OK="\tOK\t\\N\t" -FAIL="\tFAIL\t\\N\t" - -FAILURE_CONTEXT_LINES=50 -FAILURE_CONTEXT_MAX_LINE_WIDTH=400 - -function escaped() -{ - # That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language. - # Also limit lines width just in case (too long lines are not really useful usually) - clickhouse local -S 's String' --input-format=LineAsString -q "select substr(s, 1, $FAILURE_CONTEXT_MAX_LINE_WIDTH) - from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'" -} -function head_escaped() -{ - head -n $FAILURE_CONTEXT_LINES $1 | escaped -} -function unts() -{ - grep -Po "[0-9][0-9]:[0-9][0-9] \K.*" -} -function trim_server_logs() -{ - head -n $FAILURE_CONTEXT_LINES "/test_output/$1" | grep -Eo " \[ [0-9]+ \] \{.*" | escaped -} - -function install_packages() -{ - dpkg -i $1/clickhouse-common-static_*.deb - dpkg -i $1/clickhouse-common-static-dbg_*.deb - dpkg -i $1/clickhouse-server_*.deb - dpkg -i $1/clickhouse-client_*.deb -} - -function configure() -{ - # install test configs - export USE_DATABASE_ORDINARY=1 - export EXPORT_S3_STORAGE_POLICIES=1 - /usr/share/clickhouse-test/config/install.sh - - # we mount tests folder from repo to /usr/share - ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test - ln -s /usr/share/clickhouse-test/ci/download_release_packages.py /usr/bin/download_release_packages - ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag - - # avoid too slow startup - sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ - | sed "s|100000|10000|" \ - > /etc/clickhouse-server/config.d/keeper_port.xml.tmp - sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml - sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml - sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml - - # for clickhouse-server (via service) - echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment - # for clickhouse-client - export ASAN_OPTIONS='malloc_context_size=10 allocator_release_to_os_interval_ms=10000' - - # since we run clickhouse from root - sudo chown root: /var/lib/clickhouse - - # Set more frequent update period of asynchronous metrics to more frequently update information about real memory usage (less chance of OOM). - echo "1" \ - > /etc/clickhouse-server/config.d/asynchronous_metrics_update_period_s.xml - - local total_mem - total_mem=$(awk '/MemTotal/ { print $(NF-1) }' /proc/meminfo) # KiB - total_mem=$(( total_mem*1024 )) # bytes - - # Set maximum memory usage as half of total memory (less chance of OOM). - # - # But not via max_server_memory_usage but via max_memory_usage_for_user, - # so that we can override this setting and execute service queries, like: - # - hung check - # - show/drop database - # - ... - # - # So max_memory_usage_for_user will be a soft limit, and - # max_server_memory_usage will be hard limit, and queries that should be - # executed regardless memory limits will use max_memory_usage_for_user=0, - # instead of relying on max_untracked_memory - - max_server_memory_usage_to_ram_ratio=0.5 - echo "Setting max_server_memory_usage_to_ram_ratio to ${max_server_memory_usage_to_ram_ratio}" - cat > /etc/clickhouse-server/config.d/max_server_memory_usage.xml < - ${max_server_memory_usage_to_ram_ratio} - -EOL - - local max_users_mem - max_users_mem=$((total_mem*30/100)) # 30% - echo "Setting max_memory_usage_for_user=$max_users_mem and max_memory_usage for queries to 10G" - cat > /etc/clickhouse-server/users.d/max_memory_usage_for_user.xml < - - - 10G - ${max_users_mem} - - - -EOL - - cat > /etc/clickhouse-server/config.d/core.xml < - - - 107374182400 - - - $PWD - -EOL - - # Let OOM killer terminate other processes before clickhouse-server: - cat > /etc/clickhouse-server/config.d/oom_score.xml < - -1000 - -EOL - - # Analyzer is not yet ready for testing - cat > /etc/clickhouse-server/users.d/no_analyzer.xml < - - - - - - - - - - -EOL - -} - -function stop() -{ - local max_tries="${1:-90}" - local pid - # Preserve the pid, since the server can hung after the PID will be deleted. - pid="$(cat /var/run/clickhouse-server/clickhouse-server.pid)" - - clickhouse stop --max-tries "$max_tries" --do-not-kill && return - - # We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces. - echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv - kill -TERM "$(pidof gdb)" ||: - sleep 5 - echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log - timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$pid" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log - clickhouse stop --force -} - -function start() -{ - counter=0 - until clickhouse-client --query "SELECT 1" - do - if [ "$counter" -gt ${1:-120} ] - then - echo "Cannot start clickhouse-server" - rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt ||: - echo -e "Cannot start clickhouse-server$FAIL$(trim_server_logs application_errors.txt)" >> /test_output/test_results.tsv - cat /var/log/clickhouse-server/stdout.log - tail -n100 /var/log/clickhouse-server/stderr.log - tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e ' RaftInstance:' -e ' RaftInstance' | tail -n100 - break - fi - # use root to match with current uid - clickhouse start --user root >/var/log/clickhouse-server/stdout.log 2>>/var/log/clickhouse-server/stderr.log - sleep 0.5 - counter=$((counter + 1)) - done - - # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog - # and clickhouse-server can do fork-exec, for example, to run some bridge. - # Do not set nostop noprint for all signals, because some it may cause gdb to hang, - # explicitly ignore non-fatal signals that are used by server. - # Number of SIGRTMIN can be determined only in runtime. - RTMIN=$(kill -l SIGRTMIN) - echo " -set follow-fork-mode parent -handle SIGHUP nostop noprint pass -handle SIGINT nostop noprint pass -handle SIGQUIT nostop noprint pass -handle SIGPIPE nostop noprint pass -handle SIGTERM nostop noprint pass -handle SIGUSR1 nostop noprint pass -handle SIGUSR2 nostop noprint pass -handle SIG$RTMIN nostop noprint pass -info signals -continue -backtrace full -thread apply all backtrace full -info registers -disassemble /s -up -disassemble /s -up -disassemble /s -p \"done\" -detach -quit -" > script.gdb - - # FIXME Hung check may work incorrectly because of attached gdb - # 1. False positives are possible - # 2. We cannot attach another gdb to get stacktraces if some queries hung - gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log & - sleep 5 - # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s) - time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||: -} +# Stress tests and upgrade check uses similar code that was placed +# in a separate bash library. See tests/ci/stress_tests.lib +source /usr/share/clickhouse-test/ci/stress_tests.lib install_packages package_folder @@ -396,7 +180,7 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau start -./stress --hung-check --drop-databases --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \ +stress --hung-check --drop-databases --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \ && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \ || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv @@ -413,316 +197,27 @@ unset "${!THREAD_@}" start -clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \ - || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \ - && echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(trim_server_logs application_errors.txt)" \ - >> /test_output/test_results.tsv) +check_server_start stop [ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL" [ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL" +mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.final.log + # Grep logs for sanitizer asserts, crashes and other critical errors +check_logs_for_critical_errors -# Sanitizer asserts -rg -Fa "==================" /var/log/clickhouse-server/stderr.log | rg -v "in query:" >> /test_output/tmp -rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp -rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \ - && echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \ - || echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv -rm -f /test_output/tmp +tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: -# OOM -rg -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \ - && echo -e "Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \ - || echo -e "No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv - -# Logical errors -rg -Fa "Code: 49. DB::Exception: " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/logical_errors.txt \ - && echo -e "Logical error thrown (see clickhouse-server.log or logical_errors.txt)$FAIL$(head_escaped /test_output/logical_errors.txt)" >> /test_output/test_results.tsv \ - || echo -e "No logical errors$OK" >> /test_output/test_results.tsv - -# Remove file logical_errors.txt if it's empty -[ -s /test_output/logical_errors.txt ] || rm /test_output/logical_errors.txt - -# No such key errors -rg --text "Code: 499.*The specified key does not exist" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/no_such_key_errors.txt \ - && echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(trim_server_logs no_such_key_errors.txt)" >> /test_output/test_results.tsv \ - || echo -e "No lost s3 keys$OK" >> /test_output/test_results.tsv - -# Remove file no_such_key_errors.txt if it's empty -[ -s /test_output/no_such_key_errors.txt ] || rm /test_output/no_such_key_errors.txt - -# Crash -rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \ - && echo -e "Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \ - || echo -e "Not crashed$OK" >> /test_output/test_results.tsv - -# It also checks for crash without stacktrace (printed by watchdog) -rg -Fa " " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/fatal_messages.txt \ - && echo -e "Fatal message in clickhouse-server.log (see fatal_messages.txt)$FAIL$(trim_server_logs fatal_messages.txt)" >> /test_output/test_results.tsv \ - || echo -e "No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv - -# Remove file fatal_messages.txt if it's empty -[ -s /test_output/fatal_messages.txt ] || rm /test_output/fatal_messages.txt - -rg -Fa "########################################" /test_output/* > /dev/null \ - && echo -e "Killed by signal (output files)$FAIL" >> /test_output/test_results.tsv - -function get_gdb_log_context() -{ - rg -A50 -Fa " received signal " /test_output/gdb.log | head_escaped -} - -rg -Fa " received signal " /test_output/gdb.log > /dev/null \ - && echo -e "Found signal in gdb.log$FAIL$(get_gdb_log_context)" >> /test_output/test_results.tsv - -if [ "$DISABLE_BC_CHECK" -ne "1" ]; then - echo -e "Backward compatibility check\n" - - echo "Get previous release tag" - previous_release_tag=$(clickhouse-client --version | rg -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | get_previous_release_tag) - echo $previous_release_tag - - echo "Clone previous release repository" - git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository - - echo "Download clickhouse-server from the previous release" - mkdir previous_release_package_folder - - echo $previous_release_tag | download_release_packages && echo -e "Download script exit code$OK" >> /test_output/test_results.tsv \ - || echo -e "Download script failed$FAIL" >> /test_output/test_results.tsv - - mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log - for table in query_log trace_log - do - clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||: - done - - tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: - - # Check if we cloned previous release repository successfully - if ! [ "$(ls -A previous_release_repository/tests/queries)" ] - then - echo -e "Backward compatibility check: Failed to clone previous release tests$FAIL" >> /test_output/test_results.tsv - elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ] - then - echo -e "Backward compatibility check: Failed to download previous release packages$FAIL" >> /test_output/test_results.tsv - else - echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_results.tsv - echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv - - # Uninstall current packages - dpkg --remove clickhouse-client - dpkg --remove clickhouse-server - dpkg --remove clickhouse-common-static-dbg - dpkg --remove clickhouse-common-static - - rm -rf /var/lib/clickhouse/* - - # Make BC check more funny by forcing Ordinary engine for system database - mkdir /var/lib/clickhouse/metadata - echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql - - # Install previous release packages - install_packages previous_release_package_folder - - # Start server from previous release - # Previous version may not be ready for fault injections - export ZOOKEEPER_FAULT_INJECTION=0 - configure - - # Avoid "Setting s3_check_objects_after_upload is neither a builtin setting..." - rm -f /etc/clickhouse-server/users.d/enable_blobs_check.xml ||: - rm -f /etc/clickhouse-server/users.d/marks.xml ||: - - # Remove s3 related configs to avoid "there is no disk type `cache`" - rm -f /etc/clickhouse-server/config.d/storage_conf.xml ||: - rm -f /etc/clickhouse-server/config.d/azure_storage_conf.xml ||: - - # Turn on after 22.12 - rm -f /etc/clickhouse-server/config.d/compressed_marks_and_index.xml ||: - # it uses recently introduced settings which previous versions may not have - rm -f /etc/clickhouse-server/users.d/insert_keeper_retries.xml ||: - - # Turn on after 23.1 - rm -f /etc/clickhouse-server/users.d/prefetch_settings.xml ||: - - start - - clickhouse-client --query="SELECT 'Server version: ', version()" - - # Install new package before running stress test because we should use new - # clickhouse-client and new clickhouse-test. - # - # But we should leave old binary in /usr/bin/ and debug symbols in - # /usr/lib/debug/usr/bin (if any) for gdb and internal DWARF parser, so it - # will print sane stacktraces and also to avoid possible crashes. - # - # FIXME: those files can be extracted directly from debian package, but - # actually better solution will be to use different PATH instead of playing - # games with files from packages. - mv /usr/bin/clickhouse previous_release_package_folder/ - mv /usr/lib/debug/usr/bin/clickhouse.debug previous_release_package_folder/ - install_packages package_folder - mv /usr/bin/clickhouse package_folder/ - mv /usr/lib/debug/usr/bin/clickhouse.debug package_folder/ - mv previous_release_package_folder/clickhouse /usr/bin/ - mv previous_release_package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug - - mkdir tmp_stress_output - - ./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" \ - --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \ - && echo -e "Backward compatibility check: Test script exit code$OK" >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: Test script failed$FAIL" >> /test_output/test_results.tsv - rm -rf tmp_stress_output - - # We experienced deadlocks in this command in very rare cases. Let's debug it: - timeout 10m clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables" || - ( - echo "thread apply all backtrace (on select tables count)" >> /test_output/gdb.log - timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log - clickhouse stop --force - ) - - # Use bigger timeout for previous version - stop 300 - mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log - - # Start new server - mv package_folder/clickhouse /usr/bin/ - mv package_folder/clickhouse.debug /usr/lib/debug/usr/bin/clickhouse.debug - # Disable fault injections on start (we don't test them here, and it can lead to tons of requests in case of huge number of tables). - export ZOOKEEPER_FAULT_INJECTION=0 - configure - start 500 - clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \ - || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt \ - && echo -e "Backward compatibility check: Server failed to start$FAIL$(trim_server_logs bc_check_application_errors.txt)" >> /test_output/test_results.tsv) - - clickhouse-client --query="SELECT 'Server version: ', version()" - - # Let the server run for a while before checking log. - sleep 60 - - stop - mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.dirty.log - - # Error messages (we should ignore some errors) - # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.") - # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64") - # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server. - # Let's just ignore all errors from queries ("} TCPHandler: Code:", "} executeQuery: Code:") - # FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'") - # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility - echo "Check for Error messages in server log:" - rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \ - -e "Code: 236. DB::Exception: Cancelled mutating parts" \ - -e "REPLICA_IS_ALREADY_ACTIVE" \ - -e "REPLICA_ALREADY_EXISTS" \ - -e "ALL_REPLICAS_LOST" \ - -e "DDLWorker: Cannot parse DDL task query" \ - -e "RaftInstance: failed to accept a rpc connection due to error 125" \ - -e "UNKNOWN_DATABASE" \ - -e "NETWORK_ERROR" \ - -e "UNKNOWN_TABLE" \ - -e "ZooKeeperClient" \ - -e "KEEPER_EXCEPTION" \ - -e "DirectoryMonitor" \ - -e "TABLE_IS_READ_ONLY" \ - -e "Code: 1000, e.code() = 111, Connection refused" \ - -e "UNFINISHED" \ - -e "NETLINK_ERROR" \ - -e "Renaming unexpected part" \ - -e "PART_IS_TEMPORARILY_LOCKED" \ - -e "and a merge is impossible: we didn't find" \ - -e "found in queue and some source parts for it was lost" \ - -e "is lost forever." \ - -e "Unknown index: idx." \ - -e "Cannot parse string 'Hello' as UInt64" \ - -e "} TCPHandler: Code:" \ - -e "} executeQuery: Code:" \ - -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \ - -e "[Queue = DB::DynamicRuntimeQueue]: Code: 235. DB::Exception: Part" \ - -e "The set of parts restored in place of" \ - -e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \ - -e "Code: 269. DB::Exception: Destination table is myself" \ - -e "Coordination::Exception: Connection loss" \ - -e "MutateFromLogEntryTask" \ - -e "No connection to ZooKeeper, cannot get shared table ID" \ - -e "Session expired" \ - -e "TOO_MANY_PARTS" \ - -e "Container already exists" \ - /var/log/clickhouse-server/clickhouse-server.backward.dirty.log | rg -Fa "" > /test_output/bc_check_error_messages.txt \ - && echo -e "Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)$FAIL$(trim_server_logs bc_check_error_messages.txt)" \ - >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: No Error messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv - - # Remove file bc_check_error_messages.txt if it's empty - [ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt - - # Sanitizer asserts - rg -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp - rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp - rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \ - && echo -e "Backward compatibility check: Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: No sanitizer asserts$OK" >> /test_output/test_results.tsv - rm -f /test_output/tmp - - # OOM - rg -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \ - && echo -e "Backward compatibility check: Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv - - # Logical errors - echo "Check for Logical errors in server log:" - rg -Fa -A20 "Code: 49. DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \ - && echo -e "Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)$FAIL$(trim_server_logs bc_check_logical_errors.txt)" \ - >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: No logical errors$OK" >> /test_output/test_results.tsv - - # Remove file bc_check_logical_errors.txt if it's empty - [ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt - - # Crash - rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \ - && echo -e "Backward compatibility check: Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: Not crashed$OK" >> /test_output/test_results.tsv - - # It also checks for crash without stacktrace (printed by watchdog) - echo "Check for Fatal message in server log:" - rg -Fa " " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \ - && echo -e "Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)$FAIL$(trim_server_logs bc_check_fatal_messages.txt)" \ - >> /test_output/test_results.tsv \ - || echo -e "Backward compatibility check: No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv - - # Remove file bc_check_fatal_messages.txt if it's empty - [ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt - - tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||: - for table in query_log trace_log - do - clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" \ - | zstd --threads=0 > /test_output/$table.backward.tsv.zst ||: - done - fi -fi - -dmesg -T > /test_output/dmesg.log - -# OOM in dmesg -- those are real -grep -q -F -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE' /test_output/dmesg.log \ - && echo -e "OOM in dmesg$FAIL$(head_escaped /test_output/dmesg.log)" >> /test_output/test_results.tsv \ - || echo -e "No OOM in dmesg$OK" >> /test_output/test_results.tsv +collect_query_and_trace_logs mv /var/log/clickhouse-server/stderr.log /test_output/ # Write check result into check_status.tsv # Try to choose most specific error for the whole check status clickhouse-local --structure "test String, res String, time Nullable(Float32), desc String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by -(test like 'Backward compatibility check%'), -- BC check goes last (test like '%Sanitizer%') DESC, (test like '%Killed by signal%') DESC, (test like '%gdb.log%') DESC, @@ -732,14 +227,8 @@ clickhouse-local --structure "test String, res String, time Nullable(Float32), d (test like '%OOM%') DESC, (test like '%Signal 9%') DESC, (test like '%Fatal message%') DESC, -(test like '%Error message%') DESC, -(test like '%previous release%') DESC, rowNumberInAllBlocks() LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv || echo "failure\tCannot parse test_results.tsv" > /test_output/check_status.tsv [ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv -# Core dumps -find . -type f -maxdepth 1 -name 'core.*' | while read core; do - zstd --threads=0 $core - mv $core.zst /test_output/ -done +collect_core_dumps diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile new file mode 100644 index 00000000000..8e5890b81a0 --- /dev/null +++ b/docker/test/upgrade/Dockerfile @@ -0,0 +1,31 @@ +# rebuild in #33610 +# docker build -t clickhouse/upgrade-check . +ARG FROM_TAG=latest +FROM clickhouse/stateful-test:$FROM_TAG + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + bash \ + tzdata \ + fakeroot \ + debhelper \ + parallel \ + expect \ + python3 \ + python3-lxml \ + python3-termcolor \ + python3-requests \ + curl \ + sudo \ + openssl \ + netcat-openbsd \ + telnet \ + brotli \ + && apt-get clean + +COPY run.sh / + +ENV EXPORT_S3_STORAGE_POLICIES=1 + +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh new file mode 100644 index 00000000000..b93959ee039 --- /dev/null +++ b/docker/test/upgrade/run.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# shellcheck disable=SC2094 +# shellcheck disable=SC2086 +# shellcheck disable=SC2024 + +# Avoid overlaps with previous runs +dmesg --clear + +set -x + +# we mount tests folder from repo to /usr/share +ln -s /usr/share/clickhouse-test/ci/stress.py /usr/bin/stress +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test +ln -s /usr/share/clickhouse-test/ci/download_release_packages.py /usr/bin/download_release_packages +ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag + +# Stress tests and upgrade check uses similar code that was placed +# in a separate bash library. See tests/ci/stress_tests.lib +source /usr/share/clickhouse-test/ci/stress_tests.lib + +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & +./setup_minio.sh stateless # to have a proper environment + +echo "Get previous release tag" +previous_release_tag=$(dpkg --info package_folder/clickhouse-client*.deb | grep "Version: " | awk '{print $2}' | cut -f1 -d'+' | get_previous_release_tag) +echo $previous_release_tag + +echo "Clone previous release repository" +git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository + +echo "Download clickhouse-server from the previous release" +mkdir previous_release_package_folder + +echo $previous_release_tag | download_release_packages && echo -e "Download script exit code$OK" >> /test_output/test_results.tsv \ + || echo -e "Download script failed$FAIL" >> /test_output/test_results.tsv + +# Check if we cloned previous release repository successfully +if ! [ "$(ls -A previous_release_repository/tests/queries)" ] +then + echo -e 'failure\tFailed to clone previous release tests' > /test_output/check_status.tsv + exit +elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ] +then + echo -e 'failure\tFailed to download previous release packages' > /test_output/check_status.tsv + exit +fi + +echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_results.tsv +echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv + +# Make upgrade check more funny by forcing Ordinary engine for system database +mkdir /var/lib/clickhouse/metadata +echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql + +# Install previous release packages +install_packages previous_release_package_folder + +# Start server from previous release +# Let's enable S3 storage by default +export USE_S3_STORAGE_FOR_MERGE_TREE=1 +# Previous version may not be ready for fault injections +export ZOOKEEPER_FAULT_INJECTION=0 +configure + +# But we still need default disk because some tables loaded only into it +sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ + | sed "s|
s3
|
s3
default|" \ + > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml +sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml +sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml + +start + +clickhouse-client --query="SELECT 'Server version: ', version()" + +mkdir tmp_stress_output + +stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" --upgrade-check --output-folder tmp_stress_output --global-time-limit=1200 \ + && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \ + || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv + +rm -rf tmp_stress_output + +# We experienced deadlocks in this command in very rare cases. Let's debug it: +timeout 10m clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables" || +( + echo "thread apply all backtrace (on select tables count)" >> /test_output/gdb.log + timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log + clickhouse stop --force +) + +# Use bigger timeout for previous version and disable additional hang check +stop 300 false +mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.stress.log + +# Install and start new server +install_packages package_folder +# Disable fault injections on start (we don't test them here, and it can lead to tons of requests in case of huge number of tables). +export ZOOKEEPER_FAULT_INJECTION=0 +configure +start 500 +clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \ + || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \ + && echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(trim_server_logs application_errors.txt)" \ + >> /test_output/test_results.tsv) + +# Remove file application_errors.txt if it's empty +[ -s /test_output/application_errors.txt ] || rm /test_output/application_errors.txt + +clickhouse-client --query="SELECT 'Server version: ', version()" + +# Let the server run for a while before checking log. +sleep 60 + +stop +mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.upgrade.log + +# Error messages (we should ignore some errors) +# FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.") +# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64") +# FIXME Not sure if it's expected, but some tests from stress test may not be finished yet when we restarting server. +# Let's just ignore all errors from queries ("} TCPHandler: Code:", "} executeQuery: Code:") +# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'") +# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility +echo "Check for Error messages in server log:" +rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \ + -e "Code: 236. DB::Exception: Cancelled mutating parts" \ + -e "REPLICA_IS_ALREADY_ACTIVE" \ + -e "REPLICA_ALREADY_EXISTS" \ + -e "ALL_REPLICAS_LOST" \ + -e "DDLWorker: Cannot parse DDL task query" \ + -e "RaftInstance: failed to accept a rpc connection due to error 125" \ + -e "UNKNOWN_DATABASE" \ + -e "NETWORK_ERROR" \ + -e "UNKNOWN_TABLE" \ + -e "ZooKeeperClient" \ + -e "KEEPER_EXCEPTION" \ + -e "DirectoryMonitor" \ + -e "TABLE_IS_READ_ONLY" \ + -e "Code: 1000, e.code() = 111, Connection refused" \ + -e "UNFINISHED" \ + -e "NETLINK_ERROR" \ + -e "Renaming unexpected part" \ + -e "PART_IS_TEMPORARILY_LOCKED" \ + -e "and a merge is impossible: we didn't find" \ + -e "found in queue and some source parts for it was lost" \ + -e "is lost forever." \ + -e "Unknown index: idx." \ + -e "Cannot parse string 'Hello' as UInt64" \ + -e "} TCPHandler: Code:" \ + -e "} executeQuery: Code:" \ + -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \ + -e "The set of parts restored in place of" \ + -e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \ + -e "Code: 269. DB::Exception: Destination table is myself" \ + -e "Coordination::Exception: Connection loss" \ + -e "MutateFromLogEntryTask" \ + -e "No connection to ZooKeeper, cannot get shared table ID" \ + -e "Session expired" \ + -e "TOO_MANY_PARTS" \ + -e "Authentication failed" \ + -e "Container already exists" \ + /var/log/clickhouse-server/clickhouse-server.upgrade.log | zgrep -Fa "" > /test_output/upgrade_error_messages.txt \ + && echo -e "Error message in clickhouse-server.log (see upgrade_error_messages.txt)$FAIL$(head_escaped /test_output/bc_check_error_messages.txt)" \ + >> /test_output/test_results.tsv \ + || echo -e "No Error messages after server upgrade$OK" >> /test_output/test_results.tsv + +# Remove file upgrade_error_messages.txt if it's empty +[ -s /test_output/upgrade_error_messages.txt ] || rm /test_output/upgrade_error_messages.txt + +# Grep logs for sanitizer asserts, crashes and other critical errors +check_logs_for_critical_errors + +tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: + +collect_query_and_trace_logs + +check_oom_in_dmesg + +mv /var/log/clickhouse-server/stderr.log /test_output/ + +# Write check result into check_status.tsv +# Try to choose most specific error for the whole check status +clickhouse-local --structure "test String, res String, time Nullable(Float32), desc String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by +(test like '%Sanitizer%') DESC, +(test like '%Killed by signal%') DESC, +(test like '%gdb.log%') DESC, +(test ilike '%possible deadlock%') DESC, +(test like '%start%') DESC, +(test like '%dmesg%') DESC, +(test like '%OOM%') DESC, +(test like '%Signal 9%') DESC, +(test like '%Fatal message%') DESC, +(test like '%Error message%') DESC, +(test like '%previous release%') DESC, +rowNumberInAllBlocks() +LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv || echo "failure\tCannot parse test_results.tsv" > /test_output/check_status.tsv +[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv + +collect_core_dumps diff --git a/docs/changelogs/v22.3.19.6-lts.md b/docs/changelogs/v22.3.19.6-lts.md new file mode 100644 index 00000000000..d5b45f4ce66 --- /dev/null +++ b/docs/changelogs/v22.3.19.6-lts.md @@ -0,0 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.3.19.6-lts (467e0a7bd77) FIXME as compared to v22.3.18.37-lts (fe512717551) + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46440](https://github.com/ClickHouse/ClickHouse/issues/46440): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 28aee32c717..9ae49e8f707 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -39,12 +39,59 @@ To add new test, create a `.sql` or `.sh` file in `queries/0_stateless` director Tests should use (create, drop, etc) only tables in `test` database that is assumed to be created beforehand; also tests can use temporary tables. +### Restricting test runs + +A test can have zero or more _test tags_ specifying restrictions for test runs. + +For `.sh` tests tags are written as a comment on the second line: + +```bash +#!/usr/bin/env bash +# Tags: no-fasttest +``` + +For `.sql` tests tags are placed in the first line as a SQL comment: + +```sql +-- Tags: no-fasttest +SELECT 1 +``` + +|Tag name | What it does | Usage example | +|---|---|---| +| `disabled`| Test is not run || +| `long` | Test's execution time is extended from 1 to 10 minutes || +| `deadlock` | Test is run in a loop for a long time || +| `race` | Same as `deadlock`. Prefer `deadlock` || +| `shard` | Server is required to listen to `127.0.0.*` || +| `distributed` | Same as `shard`. Prefer `shard` || +| `global` | Same as `shard`. Prefer `shard` || +| `zookeeper` | Test requires Zookeeper or ClickHouse Keeper to run | Test uses `ReplicatedMergeTree` | +| `replica` | Same as `zookeeper`. Prefer `zookeeper` || +| `no-fasttest`| Test is not run under [Fast test](continuous-integration#fast-test) | Test uses `MySQL` table engine which is disabled in Fast test| +| `no-[asan, tsan, msan, ubsan]` | Disables tests in build with [sanitizers](#sanitizers) | Test is run under QEMU which doesn't work with sanitizers | +| `no-replicated-database` ||| +| `no-ordinary-database` ||| +| `no-parallel` | Disables running other tests in parallel with this one | Test reads from `system` tables and invariants may be broken| +| `no-parallel-replicas` ||| +| `no-debug` ||| +| `no-stress` ||| +| `no-polymorphic-parts` ||| +| `no-random-settings` ||| +| `no-random-merge-tree-settings` ||| +| `no-backward-compatibility-check` ||| +| `no-cpu-x86_64` ||| +| `no-cpu-aarch64` ||| +| `no-cpu-ppc64le` ||| +| `no-s3-storage` ||| + +In addition to the above settings, you can use `USE_*` flags from `system.build_options` to define usage of particular ClickHouse features. +For example, if your test uses a MySQL table, you should add a tag `use-mysql`. + ### Choosing the Test Name The name of the test starts with a five-digit prefix followed by a descriptive name, such as `00422_hash_function_constexpr.sql`. To choose the prefix, find the largest prefix already present in the directory, and increment it by one. In the meantime, some other tests might be added with the same numeric prefix, but this is OK and does not lead to any problems, you don't have to change it later. -Some tests are marked with `zookeeper`, `shard` or `long` in their names. `zookeeper` is for tests that are using ZooKeeper. `shard` is for tests that requires server to listen `127.0.0.*`; `distributed` or `global` have the same meaning. `long` is for tests that run slightly longer that one second. You can disable these groups of tests using `--no-zookeeper`, `--no-shard` and `--no-long` options, respectively. Make sure to add a proper prefix to your test name if it needs ZooKeeper or distributed queries. - ### Checking for an Error that Must Occur Sometimes you want to test that a server error occurs for an incorrect query. We support special annotations for this in SQL tests, in the following form: diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index e482926f400..f841f157376 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -1,6 +1,6 @@ # Approximate Nearest Neighbor Search Indexes [experimental] {#table_engines-ANNIndex} -The main task that indexes achieve is to quickly find nearest neighbors for multidimensional data. An example of such a problem can be finding similar pictures (texts) for a given picture (text). That problem can be reduced to finding the nearest [embeddings](https://cloud.google.com/architecture/overview-extracting-and-serving-feature-embeddings-for-machine-learning). They can be created from data using [UDF](../../../sql-reference/functions/index.md#executable-user-defined-functions). +The main task that indexes achieve is to quickly find nearest neighbors for multidimensional data. An example of such a problem can be finding similar pictures (texts) for a given picture (text). That problem can be reduced to finding the nearest [embeddings](https://cloud.google.com/architecture/overview-extracting-and-serving-feature-embeddings-for-machine-learning). They can be created from data using [UDF](/docs/en/sql-reference/functions/index.md/#executable-user-defined-functions). The next queries find the closest neighbors in N-dimensional space using the L2 (Euclidean) distance: ``` sql @@ -39,7 +39,7 @@ Approximate Nearest Neighbor Search Indexes (`ANNIndexes`) are similar to skip i LIMIT N ``` -In these queries, `DistanceFunction` is selected from [distance functions](../../../sql-reference/functions/distance-functions). `Point` is a known vector (something like `(0.1, 0.1, ... )`). To avoid writing large vectors, use [client parameters](../../../interfaces/cli.md#queries-with-parameters-cli-queries-with-parameters). `Value` - a float value that will bound the neighbourhood. +In these queries, `DistanceFunction` is selected from [distance functions](/docs/en/sql-reference/functions/distance-functions.md). `Point` is a known vector (something like `(0.1, 0.1, ... )`). To avoid writing large vectors, use [client parameters](/docs/en//interfaces/cli.md#queries-with-parameters-cli-queries-with-parameters). `Value` - a float value that will bound the neighbourhood. :::note ANN index can't speed up query that satisfies both types (`where + order by`, only one of them). All queries must have the limit, as algorithms are used to find nearest neighbors and need a specific number of them. @@ -85,13 +85,13 @@ As the indexes are built only during insertions into table, `INSERT` and `OPTIMI You can create your table with index which uses certain algorithm. Now only indices based on the following algorithms are supported: # Index list -- [Annoy](../../../engines/table-engines/mergetree-family/annindexes.md#annoy-annoy) +- [Annoy](/docs/en/engines/table-engines/mergetree-family/annindexes.md#annoy-annoy) # Annoy {#annoy} Implementation of the algorithm was taken from [this repository](https://github.com/spotify/annoy). Short description of the algorithm: -The algorithm recursively divides in half all space by random linear surfaces (lines in 2D, planes in 3D e.t.c.). Thus it makes tree of polyhedrons and points that they contains. Repeating the operation several times for greater accuracy it creates a forest. +The algorithm recursively divides in half all space by random linear surfaces (lines in 2D, planes in 3D etc.). Thus it makes tree of polyhedrons and points that they contains. Repeating the operation several times for greater accuracy it creates a forest. To find K Nearest Neighbours it goes down through the trees and fills the buffer of closest points using the priority queue of polyhedrons. Next, it sorts buffer and return the nearest K points. __Examples__: @@ -118,7 +118,7 @@ ORDER BY id; ``` :::note -Table with array field will work faster, but all arrays **must** have same length. Use [CONSTRAINT](../../../sql-reference/statements/create/table.md#constraints) to avoid errors. For example, `CONSTRAINT constraint_name_1 CHECK length(data) = 256`. +Table with array field will work faster, but all arrays **must** have same length. Use [CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints) to avoid errors. For example, `CONSTRAINT constraint_name_1 CHECK length(data) = 256`. ::: Parameter `NumTrees` is the number of trees which the algorithm will create. The bigger it is, the slower (approximately linear) it works (in both `CREATE` and `SELECT` requests), but the better accuracy you get (adjusted for randomness). By default it is set to `100`. Parameter `DistanceName` is name of distance function. By default it is set to `L2Distance`. It can be set without changing first parameter, for example diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 992eaa8a49f..b2b2c6d5b1e 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1971,7 +1971,8 @@ To exchange data with Hadoop, you can use [HDFS table engine](/docs/en/engines/t - [input_format_parquet_case_insensitive_column_matching](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_case_insensitive_column_matching) - ignore case when matching Parquet columns with ClickHouse columns. Default value - `false`. - [input_format_parquet_allow_missing_columns](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_allow_missing_columns) - allow missing columns while reading Parquet data. Default value - `false`. - [input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference) - allow skipping columns with unsupported types while schema inference for Parquet format. Default value - `false`. -- [output_format_parquet_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_fixed_string_as_fixed_byte_array) - use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary/String for FixedString columns. Default value - `true`. +- [output_format_parquet_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_fixed_string_as_fixed_byte_array) - use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary/String for FixedString columns. Default value - `true`. +- [output_format_parquet_version](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_version) - The version of Parquet format used in output format. Default value - `2.latest`. ## Arrow {#data-format-arrow} diff --git a/docs/en/interfaces/schema-inference.md b/docs/en/interfaces/schema-inference.md index 2b50794c458..25bdb0c36a3 100644 --- a/docs/en/interfaces/schema-inference.md +++ b/docs/en/interfaces/schema-inference.md @@ -229,7 +229,7 @@ To prevent inferring the same schema every time ClickHouse read the data from th There are special settings that control this cache: - `schema_inference_cache_max_elements_for_{file/s3/hdfs/url}` - the maximum number of cached schemas for the corresponding table function. The default value is `4096`. These settings should be set in the server config. -- `use_cache_for_{file,s3,hdfs,url}_schema_inference` - allows turning on/off using cache for schema inference. These settings can be used in queries. +- `schema_inference_use_cache_for_{file,s3,hdfs,url}` - allows turning on/off using cache for schema inference. These settings can be used in queries. The schema of the file can be changed by modifying the data or by changing format settings. For this reason, the schema inference cache identifies the schema by file source, format name, used format settings, and the last modification time of the file. diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index 87c51940c73..83ef46053a4 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -50,7 +50,7 @@ If there are multiple profiles active for a user, then constraints are merged. M Read-only mode is enabled by `readonly` setting (not to confuse with `readonly` constraint type): - `readonly=0`: No read-only restrictions. -- `readonly=1`: Only read queries are allowed and settings cannot be changes unless `changeable_in_readonly` is set. +- `readonly=1`: Only read queries are allowed and settings cannot be changed unless `changeable_in_readonly` is set. - `readonly=2`: Only read queries are allowed, but settings can be changed, except for `readonly` setting itself. diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 57d1a8448cc..3580d83f704 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -142,6 +142,10 @@ y Nullable(String) z IPv4 ``` +:::warning +If the `schema_inference_hints` is not formated properly, or if there is a typo or a wrong datatype, etc... the whole schema_inference_hints will be ignored. +::: + ## schema_inference_make_columns_nullable {#schema_inference_make_columns_nullable} Controls making inferred types `Nullable` in schema inference for formats without information about nullability. @@ -1102,6 +1106,12 @@ Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary/String for FixedStrin Enabled by default. +### output_format_parquet_version {#output_format_parquet_version} + +The version of Parquet format used in output format. Supported versions: `1.0`, `2.4`, `2.6` and `2.latest`. + +Default value: `2.latest`. + ## Hive format settings {#hive-format-settings} ### input_format_hive_text_fields_delimiter {#input_format_hive_text_fields_delimiter} diff --git a/docs/en/operations/system-tables/server_settings.md b/docs/en/operations/system-tables/server_settings.md new file mode 100644 index 00000000000..e1bf8c3d63f --- /dev/null +++ b/docs/en/operations/system-tables/server_settings.md @@ -0,0 +1,52 @@ +--- +slug: /en/operations/system-tables/server_settings +--- +# server_settings + +Contains information about global settings for the server, which were specified in `config.xml`. +Currently, the table shows only settings from the first layer of `config.xml` and doesn't support nested configs (e.g. [logger](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-logger)). + +Columns: + +- `name` ([String](../../sql-reference/data-types/string.md)) — Server setting name. +- `value` ([String](../../sql-reference/data-types/string.md)) — Server setting value. +- `default` ([String](../../sql-reference/data-types/string.md)) — Server setting default value. +- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether a setting was specified in `config.xml` +- `description` ([String](../../sql-reference/data-types/string.md)) — Short server setting description. +- `type` ([String](../../sql-reference/data-types/string.md)) — Server setting value type. + +**Example** + +The following example shows how to get information about server settings which name contains `thread_pool`. + +``` sql +SELECT * +FROM system.server_settings +WHERE name LIKE '%thread_pool%' +``` + +``` text +┌─name─────────────────────────┬─value─┬─default─┬─changed─┬─description─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─type───┐ +│ max_thread_pool_size │ 5000 │ 10000 │ 1 │ The maximum number of threads that could be allocated from the OS and used for query execution and background operations. │ UInt64 │ +│ max_thread_pool_free_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks. │ UInt64 │ +│ thread_pool_queue_size │ 10000 │ 10000 │ 0 │ The maximum number of tasks that will be placed in a queue and wait for execution. │ UInt64 │ +│ max_io_thread_pool_size │ 100 │ 100 │ 0 │ The maximum number of threads that would be used for IO operations │ UInt64 │ +│ max_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for IO thread pool. │ UInt64 │ +│ io_thread_pool_queue_size │ 10000 │ 10000 │ 0 │ Queue size for IO thread pool. │ UInt64 │ +└──────────────────────────────┴───────┴─────────┴─────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴────────┘ +``` + +Using of `WHERE changed` can be useful, for example, when you want to check +whether settings in configuration files are loaded correctly and are in use. + + + +``` sql +SELECT * FROM system.server_settings WHERE changed AND name='max_thread_pool_size' +``` + +**See also** + +- [Settings](../../operations/system-tables/settings.md) +- [Configuration Files](../../operations/configuration-files.md) +- [Server Settings](../../operations/server-configuration-parameters/settings.md) diff --git a/docs/en/operations/system-tables/settings.md b/docs/en/operations/system-tables/settings.md index 8ce74ea3ae3..a3dfa937abe 100644 --- a/docs/en/operations/system-tables/settings.md +++ b/docs/en/operations/system-tables/settings.md @@ -16,6 +16,7 @@ Columns: - `readonly` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Shows whether the current user can change the setting: - `0` — Current user can change the setting. - `1` — Current user can’t change the setting. +- `default` ([String](../../sql-reference/data-types/string.md)) — Setting default value. **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md b/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md index 4d29e6f1272..bdcc6c0e3d0 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md +++ b/docs/en/sql-reference/aggregate-functions/reference/covarsamp.md @@ -7,8 +7,8 @@ sidebar_position: 37 Calculates the value of `Σ((x - x̅)(y - y̅)) / (n - 1)`. -Returns Float64. When `n <= 1`, returns +∞. +Returns Float64. When `n <= 1`, returns `nan`. :::note This function uses a numerically unstable algorithm. If you need [numerical stability](https://en.wikipedia.org/wiki/Numerical_stability) in calculations, use the `covarSampStable` function. It works slower but provides a lower computational error. -::: \ No newline at end of file +::: diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md index a4479fd8589..c5244cf62e3 100644 --- a/docs/en/sql-reference/functions/arithmetic-functions.md +++ b/docs/en/sql-reference/functions/arithmetic-functions.md @@ -48,7 +48,35 @@ When dividing by zero you get ‘inf’, ‘-inf’, or ‘nan’. ## intDiv(a, b) Calculates the quotient of the numbers. Divides into integers, rounding down (by the absolute value). -An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. + +Returns an integer of the type of the dividend (the first parameter). + +An exception is thrown when dividing by zero, when the quotient does not fit in the range of the dividend, or when dividing a minimal negative number by minus one. + +**Example** + +Query: + +```sql +SELECT + intDiv(toFloat64(1), 0.001) AS res, + toTypeName(res) +``` +```response +┌──res─┬─toTypeName(intDiv(toFloat64(1), 0.001))─┐ +│ 1000 │ Int64 │ +└──────┴─────────────────────────────────────────┘ +``` + +```sql +SELECT + intDiv(1, 0.001) AS res, + toTypeName(res) +``` +```response +Received exception from server (version 23.2.1): +Code: 153. DB::Exception: Received from localhost:9000. DB::Exception: Cannot perform integer division, because it will produce infinite or too large number: While processing intDiv(1, 0.001) AS res, toTypeName(res). (ILLEGAL_DIVISION) +``` ## intDivOrZero(a, b) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index bcd118ce0be..10bc73c4a72 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -579,3 +579,33 @@ Result: │ 3628800 │ └───────────────┘ ``` + +## width_bucket(operand, low, high, count) + +Returns the number of the bucket in which `operand` falls in a histogram having `count` equal-width buckets spanning the range `low` to `high`. Returns `0` if `operand < low`, and returns `count+1` if `operand >= high`. + +`operand`, `low`, `high` can be any native number type. `count` can only be unsigned native integer and its value cannot be zero. + +**Syntax** + +```sql +widthBucket(operand, low, high, count) +``` + +There is also a case insensitive alias called `WIDTH_BUCKET` to provide compatibility with other databases. + +**Example** + +Query: + +``` sql +SELECT widthBucket(10.15, -8.6, 23, 18); +``` + +Result: + +``` text +┌─widthBucket(10.15, -8.6, 23, 18)─┐ +│ 11 │ +└──────────────────────────────────┘ +``` \ No newline at end of file diff --git a/docs/en/sql-reference/functions/splitting-merging-functions.md b/docs/en/sql-reference/functions/splitting-merging-functions.md index 7cad6b2fbbf..6015bb79b87 100644 --- a/docs/en/sql-reference/functions/splitting-merging-functions.md +++ b/docs/en/sql-reference/functions/splitting-merging-functions.md @@ -226,6 +226,17 @@ SELECT splitByNonAlpha(' 1! a, b. '); Concatenates string representations of values listed in the array with the separator. `separator` is an optional parameter: a constant string, set to an empty string by default. Returns the string. +**Example** + +``` sql +SELECT arrayStringConcat(['12/05/2021', '12:50:00'], ' ') AS DateString; +``` +```text +┌─DateString──────────┐ +│ 12/05/2021 12:50:00 │ +└─────────────────────┘ +``` + ## alphaTokens(s[, max_substrings]), splitByAlpha(s[, max_substrings]) Selects substrings of consecutive bytes from the ranges a-z and A-Z.Returns an array of substrings. @@ -364,4 +375,4 @@ Result: ┌─tokens────────────────────────────┐ │ ['test1','test2','test3','test4'] │ └───────────────────────────────────┘ -``` \ No newline at end of file +``` diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md index c4ca89f3284..c2424ff6046 100644 --- a/docs/en/sql-reference/statements/create/settings-profile.md +++ b/docs/en/sql-reference/statements/create/settings-profile.md @@ -19,8 +19,15 @@ CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_n ## Example +Create a user: +```sql +CREATE USER robin IDENTIFIED BY 'password'; +``` + Create the `max_memory_usage_profile` settings profile with value and constraints for the `max_memory_usage` setting and assign it to user `robin`: ``` sql -CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin +CREATE +SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 +TO robin ``` diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 9e66afba613..54977e1b0ab 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -17,10 +17,11 @@ By default, tables are created only on the current server. Distributed DDL queri ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( - name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr1] [compression_codec] [TTL expr1], - name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [compression_codec] [TTL expr2], + name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr1] [compression_codec] [TTL expr1] [COMMENT 'comment for column'], + name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [compression_codec] [TTL expr2] [COMMENT 'comment for column'], ... ) ENGINE = engine + COMMENT 'comment for table' ``` Creates a table named `table_name` in the `db` database or the current database if `db` is not set, with the structure specified in brackets and the `engine` engine. @@ -32,6 +33,8 @@ Expressions can also be defined for default values (see below). If necessary, primary key can be specified, with one or more key expressions. +Comments can be added for columns and for the table. + ### With a Schema Similar to Other Table ``` sql @@ -267,7 +270,7 @@ You can define a [primary key](../../../engines/table-engines/mergetree-family/m CREATE TABLE db.table_name ( name1 type1, name2 type2, ..., - PRIMARY KEY(expr1[, expr2,...])] + PRIMARY KEY(expr1[, expr2,...]) ) ENGINE = engine; ``` diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 527b31b36a4..acdede3c673 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -54,6 +54,10 @@ SELECT * FROM view(column1=value1, column2=value2 ...) CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]name] [ENGINE = engine] [POPULATE] AS SELECT ... ``` +:::tip +Here is a step by step guide on using [Materialized views](docs/en/guides/developer/cascading-materialized-views.md). +::: + Materialized views store data transformed by the corresponding [SELECT](../../../sql-reference/statements/select/index.md) query. When creating a materialized view without `TO [db].[table]`, you must specify `ENGINE` – the table engine for storing data. diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index 94b23bc695c..b49c2f8da20 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -23,23 +23,3 @@ You can use table functions in: :::warning You can’t use table functions if the [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) setting is disabled. ::: - -| Function | Description | -|------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| -| [file](../../sql-reference/table-functions/file.md) | Creates a [File](../../engines/table-engines/special/file.md)-engine table. | -| [merge](../../sql-reference/table-functions/merge.md) | Creates a [Merge](../../engines/table-engines/special/merge.md)-engine table. | -| [numbers](../../sql-reference/table-functions/numbers.md) | Creates a table with a single column filled with integer numbers. | -| [remote](../../sql-reference/table-functions/remote.md) | Allows you to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md)-engine table. | -| [url](../../sql-reference/table-functions/url.md) | Creates a [Url](../../engines/table-engines/special/url.md)-engine table. | -| [mysql](../../sql-reference/table-functions/mysql.md) | Creates a [MySQL](../../engines/table-engines/integrations/mysql.md)-engine table. | -| [postgresql](../../sql-reference/table-functions/postgresql.md) | Creates a [PostgreSQL](../../engines/table-engines/integrations/postgresql.md)-engine table. | -| [jdbc](../../sql-reference/table-functions/jdbc.md) | Creates a [JDBC](../../engines/table-engines/integrations/jdbc.md)-engine table. | -| [odbc](../../sql-reference/table-functions/odbc.md) | Creates a [ODBC](../../engines/table-engines/integrations/odbc.md)-engine table. | -| [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a [HDFS](../../engines/table-engines/integrations/hdfs.md)-engine table. | -| [s3](../../sql-reference/table-functions/s3.md) | Creates a [S3](../../engines/table-engines/integrations/s3.md)-engine table. | -| [sqlite](../../sql-reference/table-functions/sqlite.md) | Creates a [sqlite](../../engines/table-engines/integrations/sqlite.md)-engine table. | - -:::note -Only these table functions are enabled in readonly mode : -null, view, viewIfPermitted, numbers, numbers_mt, generateRandom, values, cluster, clusterAllReplicas -::: \ No newline at end of file diff --git a/docs/ru/operations/system-tables/server_settings.md b/docs/ru/operations/system-tables/server_settings.md new file mode 100644 index 00000000000..4c0f4af1da2 --- /dev/null +++ b/docs/ru/operations/system-tables/server_settings.md @@ -0,0 +1,53 @@ +--- +slug: /ru/operations/system-tables/server_settings +--- +# system.server_settings + +Содержит информацию о конфигурации сервера. +В настоящий момент таблица содержит только верхнеуровневые параметры из файла `config.xml` и не поддерживает вложенные конфигурации +(например [logger](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-logger)) + +Столбцы: + +- `name` ([String](../../sql-reference/data-types/string.md)) — имя настройки. +- `value` ([String](../../sql-reference/data-types/string.md)) — значение настройки. +- `default` ([String](../../sql-reference/data-types/string.md)) — значению настройки по умолчанию. +- `changed` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — показывает, была ли настройка указана в `config.xml` или является значением по-умолчанию. +- `description` ([String](../../sql-reference/data-types/string.md)) — краткое описание настройки. +- `type` ([String](../../sql-reference/data-types/string.md)) — тип настройки. + +**Пример** + +Пример показывает как получить информацию о настройках, имена которых содержат `thread_pool`. + +``` sql +SELECT * +FROM system.server_settings +WHERE name LIKE '%thread_pool%' +``` + +``` text +┌─name─────────────────────────┬─value─┬─default─┬─changed─┬─description─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─type───┐ +│ max_thread_pool_size │ 5000 │ 10000 │ 1 │ The maximum number of threads that could be allocated from the OS and used for query execution and background operations. │ UInt64 │ +│ max_thread_pool_free_size │ 1000 │ 1000 │ 0 │ The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks. │ UInt64 │ +│ thread_pool_queue_size │ 10000 │ 10000 │ 0 │ The maximum number of tasks that will be placed in a queue and wait for execution. │ UInt64 │ +│ max_io_thread_pool_size │ 100 │ 100 │ 0 │ The maximum number of threads that would be used for IO operations │ UInt64 │ +│ max_io_thread_pool_free_size │ 0 │ 0 │ 0 │ Max free size for IO thread pool. │ UInt64 │ +│ io_thread_pool_queue_size │ 10000 │ 10000 │ 0 │ Queue size for IO thread pool. │ UInt64 │ +└──────────────────────────────┴───────┴─────────┴─────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴────────┘ +``` + +Использование `WHERE changed` может быть полезно, например, если необходимо проверить, +что настройки корректно загрузились из конфигурационного файла и используются. + + + +``` sql +SELECT * FROM system.settings WHERE changed AND name='max_thread_pool_size' +``` + +**Cм. также** + +- [Настройки](../../operations/system-tables/settings.md) +- [Конфигурационные файлы](../../operations/configuration-files.md) +- [Настройки сервера](../../operations/server-configuration-parameters/settings.md) diff --git a/docs/ru/operations/system-tables/settings.md b/docs/ru/operations/system-tables/settings.md index 09802415054..31eb77d4d41 100644 --- a/docs/ru/operations/system-tables/settings.md +++ b/docs/ru/operations/system-tables/settings.md @@ -16,6 +16,7 @@ slug: /ru/operations/system-tables/settings - `readonly` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Показывает, может ли пользователь изменять настройку: - `0` — Текущий пользователь может изменять настройку. - `1` — Текущий пользователь не может изменять настройку. +- `default` ([String](../../sql-reference/data-types/string.md)) — значению настройки по умолчанию. **Пример** diff --git a/packages/clickhouse-server.service b/packages/clickhouse-server.service index 037be826b97..090461df988 100644 --- a/packages/clickhouse-server.service +++ b/packages/clickhouse-server.service @@ -18,12 +18,14 @@ Group=clickhouse Restart=always RestartSec=30 # Since ClickHouse is systemd aware default 1m30sec may not be enough -TimeoutStartSec=inifinity +TimeoutStartSec=infinity # %p is resolved to the systemd unit name RuntimeDirectory=%p ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=%t/%p/%p.pid # Minus means that this file is optional. EnvironmentFile=-/etc/default/%p +# Bring back /etc/default/clickhouse for backward compatibility +EnvironmentFile=-/etc/default/clickhouse LimitCORE=infinity LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8c60f840b89..1486a51c710 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -91,6 +91,7 @@ #include #include #include +#include #include #include @@ -662,7 +663,10 @@ try MainThreadStatus::getInstance(); - StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); + ServerSettings server_settings; + server_settings.loadSettingsFromConfig(config()); + + StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces); #if USE_HDFS /// This will point libhdfs3 to the right location for its config. @@ -696,7 +700,7 @@ try { const String config_path = config().getString("config-file", "config.xml"); const auto config_dir = std::filesystem::path{config_path}.replace_filename("openssl.conf"); - setenv("OPENSSL_CONF", config_dir.string(), true); + setenv("OPENSSL_CONF", config_dir.c_str(), true); } #endif @@ -747,9 +751,9 @@ try // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well. GlobalThreadPool::initialize( - config().getUInt("max_thread_pool_size", 10000), - config().getUInt("max_thread_pool_free_size", 1000), - config().getUInt("thread_pool_queue_size", 10000)); + server_settings.max_thread_pool_size, + server_settings.max_thread_pool_free_size, + server_settings.thread_pool_queue_size); #if USE_AZURE_BLOB_STORAGE /// It makes sense to deinitialize libxml after joining of all threads @@ -765,9 +769,9 @@ try #endif IOThreadPool::initialize( - config().getUInt("max_io_thread_pool_size", 100), - config().getUInt("max_io_thread_pool_free_size", 0), - config().getUInt("io_thread_pool_queue_size", 10000)); + server_settings.max_io_thread_pool_size, + server_settings.max_io_thread_pool_free_size, + server_settings.io_thread_pool_queue_size); /// Initialize global local cache for remote filesystem. if (config().has("local_cache_for_remote_fs")) @@ -783,15 +787,15 @@ try } } - Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); + Poco::ThreadPool server_pool(3, server_settings.max_connections); std::mutex servers_lock; std::vector servers; std::vector servers_to_start_before_tables; /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, - config().getUInt("asynchronous_metrics_update_period_s", 1), - config().getUInt("asynchronous_heavy_metrics_update_period_s", 120), + server_settings.asynchronous_metrics_update_period_s, + server_settings.asynchronous_heavy_metrics_update_period_s, [&]() -> std::vector { std::vector metrics; @@ -806,7 +810,7 @@ try } ); - ConnectionCollector::init(global_context, config().getUInt("max_threads_for_connection_collector", 10)); + ConnectionCollector::init(global_context, server_settings.max_threads_for_connection_collector); bool has_zookeeper = config().has("zookeeper"); @@ -825,6 +829,9 @@ try Settings::checkNoSettingNamesAtTopLevel(config(), config_path); + /// We need to reload server settings because config could be updated via zookeeper. + server_settings.loadSettingsFromConfig(config()); + #if defined(OS_LINUX) std::string executable_path = getExecutablePath(); @@ -944,7 +951,7 @@ try std::string path_str = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); fs::path path = path_str; - std::string default_database = config().getString("default_database", "default"); + std::string default_database = server_settings.default_database.toString(); /// Check that the process user id matches the owner of the data. const auto effective_user_id = geteuid(); @@ -1035,21 +1042,18 @@ try LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); /// Storage with temporary data for processing of heavy queries. - if (auto temporary_policy = config().getString("tmp_policy", ""); !temporary_policy.empty()) + if (!server_settings.tmp_policy.value.empty()) { - size_t max_size = config().getUInt64("max_temporary_data_on_disk_size", 0); - global_context->setTemporaryStoragePolicy(temporary_policy, max_size); + global_context->setTemporaryStoragePolicy(server_settings.tmp_policy, server_settings.max_temporary_data_on_disk_size); } - else if (auto temporary_cache = config().getString("temporary_data_in_cache", ""); !temporary_cache.empty()) + else if (!server_settings.temporary_data_in_cache.value.empty()) { - size_t max_size = config().getUInt64("max_temporary_data_on_disk_size", 0); - global_context->setTemporaryStorageInCache(temporary_cache, max_size); + global_context->setTemporaryStorageInCache(server_settings.temporary_data_in_cache, server_settings.max_temporary_data_on_disk_size); } else { std::string temporary_path = config().getString("tmp_path", path / "tmp/"); - size_t max_size = config().getUInt64("max_temporary_data_on_disk_size", 0); - global_context->setTemporaryStoragePath(temporary_path, max_size); + global_context->setTemporaryStoragePath(temporary_path, server_settings.max_temporary_data_on_disk_size); } /** Directory with 'flags': files indicating temporary settings for the server set by system administrator. @@ -1184,10 +1188,12 @@ try { Settings::checkNoSettingNamesAtTopLevel(*config, config_path); - /// Limit on total memory usage - size_t max_server_memory_usage = config->getUInt64("max_server_memory_usage", 0); + ServerSettings server_settings; + server_settings.loadSettingsFromConfig(*config); - double max_server_memory_usage_to_ram_ratio = config->getDouble("max_server_memory_usage_to_ram_ratio", 0.9); + size_t max_server_memory_usage = server_settings.max_server_memory_usage; + + double max_server_memory_usage_to_ram_ratio = server_settings.max_server_memory_usage_to_ram_ratio; size_t default_max_server_memory_usage = static_cast(memory_amount * max_server_memory_usage_to_ram_ratio); if (max_server_memory_usage == 0) @@ -1215,8 +1221,7 @@ try total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); - bool allow_use_jemalloc_memory = config->getBool("allow_use_jemalloc_memory", true); - total_memory_tracker.setAllowUseJemallocMemory(allow_use_jemalloc_memory); + total_memory_tracker.setAllowUseJemallocMemory(server_settings.allow_use_jemalloc_memory); auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); @@ -1234,36 +1239,23 @@ try global_context->setRemoteHostFilter(*config); - /// Setup protection to avoid accidental DROP for big tables (that are greater than 50 GB by default) - if (config->has("max_table_size_to_drop")) - global_context->setMaxTableSizeToDrop(config->getUInt64("max_table_size_to_drop")); - - if (config->has("max_partition_size_to_drop")) - global_context->setMaxPartitionSizeToDrop(config->getUInt64("max_partition_size_to_drop")); + global_context->setMaxTableSizeToDrop(server_settings.max_table_size_to_drop); + global_context->setMaxPartitionSizeToDrop(server_settings.max_partition_size_to_drop); ConcurrencyControl::SlotCount concurrent_threads_soft_limit = ConcurrencyControl::Unlimited; - if (config->has("concurrent_threads_soft_limit_num")) + if (server_settings.concurrent_threads_soft_limit_num > 0 && server_settings.concurrent_threads_soft_limit_num < concurrent_threads_soft_limit) + concurrent_threads_soft_limit = server_settings.concurrent_threads_soft_limit_num; + if (server_settings.concurrent_threads_soft_limit_ratio_to_cores > 0) { - auto value = config->getUInt64("concurrent_threads_soft_limit_num", 0); - if (value > 0 && value < concurrent_threads_soft_limit) - concurrent_threads_soft_limit = value; - } - if (config->has("concurrent_threads_soft_limit_ratio_to_cores")) - { - auto value = config->getUInt64("concurrent_threads_soft_limit_ratio_to_cores", 0) * std::thread::hardware_concurrency(); + auto value = server_settings.concurrent_threads_soft_limit_ratio_to_cores * std::thread::hardware_concurrency(); if (value > 0 && value < concurrent_threads_soft_limit) concurrent_threads_soft_limit = value; } ConcurrencyControl::instance().setMaxConcurrency(concurrent_threads_soft_limit); - if (config->has("max_concurrent_queries")) - global_context->getProcessList().setMaxSize(config->getInt("max_concurrent_queries", 0)); - - if (config->has("max_concurrent_insert_queries")) - global_context->getProcessList().setMaxInsertQueriesAmount(config->getInt("max_concurrent_insert_queries", 0)); - - if (config->has("max_concurrent_select_queries")) - global_context->getProcessList().setMaxSelectQueriesAmount(config->getInt("max_concurrent_select_queries", 0)); + global_context->getProcessList().setMaxSize(server_settings.max_concurrent_queries); + global_context->getProcessList().setMaxInsertQueriesAmount(server_settings.max_concurrent_insert_queries); + global_context->getProcessList().setMaxSelectQueriesAmount(server_settings.max_concurrent_select_queries); if (config->has("keeper_server")) global_context->updateKeeperConfiguration(*config); @@ -1272,56 +1264,36 @@ try /// Note: If you specified it in the top level config (not it config of default profile) /// then ClickHouse will use it exactly. /// This is done for backward compatibility. - if (global_context->areBackgroundExecutorsInitialized() && (config->has("background_pool_size") || config->has("background_merges_mutations_concurrency_ratio"))) + if (global_context->areBackgroundExecutorsInitialized()) { - auto new_pool_size = config->getUInt64("background_pool_size", 16); - auto new_ratio = config->getUInt64("background_merges_mutations_concurrency_ratio", 2); + auto new_pool_size = server_settings.background_pool_size; + auto new_ratio = server_settings.background_merges_mutations_concurrency_ratio; global_context->getMergeMutateExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size * new_ratio); - auto new_scheduling_policy = config->getString("background_merges_mutations_scheduling_policy", "round_robin"); - global_context->getMergeMutateExecutor()->updateSchedulingPolicy(new_scheduling_policy); + global_context->getMergeMutateExecutor()->updateSchedulingPolicy(server_settings.background_merges_mutations_scheduling_policy.toString()); } - if (global_context->areBackgroundExecutorsInitialized() && config->has("background_move_pool_size")) + if (global_context->areBackgroundExecutorsInitialized()) { - auto new_pool_size = config->getUInt64("background_move_pool_size"); + auto new_pool_size = server_settings.background_move_pool_size; global_context->getMovesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size); } - if (global_context->areBackgroundExecutorsInitialized() && config->has("background_fetches_pool_size")) + if (global_context->areBackgroundExecutorsInitialized()) { - auto new_pool_size = config->getUInt64("background_fetches_pool_size"); + auto new_pool_size = server_settings.background_fetches_pool_size; global_context->getFetchesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size); } - if (global_context->areBackgroundExecutorsInitialized() && config->has("background_common_pool_size")) + if (global_context->areBackgroundExecutorsInitialized()) { - auto new_pool_size = config->getUInt64("background_common_pool_size"); + auto new_pool_size = server_settings.background_common_pool_size; global_context->getCommonExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size); } - if (config->has("background_buffer_flush_schedule_pool_size")) - { - auto new_pool_size = config->getUInt64("background_buffer_flush_schedule_pool_size"); - global_context->getBufferFlushSchedulePool().increaseThreadsCount(new_pool_size); - } - - if (config->has("background_schedule_pool_size")) - { - auto new_pool_size = config->getUInt64("background_schedule_pool_size"); - global_context->getSchedulePool().increaseThreadsCount(new_pool_size); - } - - if (config->has("background_message_broker_schedule_pool_size")) - { - auto new_pool_size = config->getUInt64("background_message_broker_schedule_pool_size"); - global_context->getMessageBrokerSchedulePool().increaseThreadsCount(new_pool_size); - } - - if (config->has("background_distributed_schedule_pool_size")) - { - auto new_pool_size = config->getUInt64("background_distributed_schedule_pool_size"); - global_context->getDistributedSchedulePool().increaseThreadsCount(new_pool_size); - } + global_context->getBufferFlushSchedulePool().increaseThreadsCount(server_settings.background_buffer_flush_schedule_pool_size); + global_context->getSchedulePool().increaseThreadsCount(server_settings.background_schedule_pool_size); + global_context->getMessageBrokerSchedulePool().increaseThreadsCount(server_settings.background_message_broker_schedule_pool_size); + global_context->getDistributedSchedulePool().increaseThreadsCount(server_settings.background_distributed_schedule_pool_size); if (config->has("resources")) { @@ -1466,18 +1438,15 @@ try }); /// Limit on total number of concurrently executed queries. - global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0)); + global_context->getProcessList().setMaxSize(server_settings.max_concurrent_queries); /// Set up caches. - /// Lower cache size on low-memory systems. - double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio", 0.5); - size_t max_cache_size = static_cast(memory_amount * cache_size_to_ram_max_ratio); + size_t max_cache_size = static_cast(memory_amount * server_settings.cache_size_to_ram_max_ratio); - /// Size of cache for uncompressed blocks. Zero means disabled. - String uncompressed_cache_policy = config().getString("uncompressed_cache_policy", "SLRU"); + String uncompressed_cache_policy = server_settings.uncompressed_cache_policy; LOG_INFO(log, "Uncompressed cache policy name {}", uncompressed_cache_policy); - size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0); + size_t uncompressed_cache_size = server_settings.uncompressed_cache_size; if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; @@ -1499,9 +1468,8 @@ try global_context, settings.async_insert_threads)); - /// Size of cache for marks (index of MergeTree family of tables). - size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120); - String mark_cache_policy = config().getString("mark_cache_policy", "SLRU"); + size_t mark_cache_size = server_settings.mark_cache_size; + String mark_cache_policy = server_settings.mark_cache_policy; if (!mark_cache_size) LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation."); if (mark_cache_size > max_cache_size) @@ -1512,20 +1480,14 @@ try } global_context->setMarkCache(mark_cache_size, mark_cache_policy); - /// Size of cache for uncompressed blocks of MergeTree indices. Zero means disabled. - size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", 0); - if (index_uncompressed_cache_size) - global_context->setIndexUncompressedCache(index_uncompressed_cache_size); + if (server_settings.index_uncompressed_cache_size) + global_context->setIndexUncompressedCache(server_settings.index_uncompressed_cache_size); - /// Size of cache for index marks (index of MergeTree skip indices). - size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", 0); - if (index_mark_cache_size) - global_context->setIndexMarkCache(index_mark_cache_size); + if (server_settings.index_mark_cache_size) + global_context->setIndexMarkCache(server_settings.index_mark_cache_size); - /// A cache for mmapped files. - size_t mmap_cache_size = config().getUInt64("mmap_cache_size", 1000); /// The choice of default is arbitrary. - if (mmap_cache_size) - global_context->setMMappedFileCache(mmap_cache_size); + if (server_settings.mmap_cache_size) + global_context->setMMappedFileCache(server_settings.mmap_cache_size); /// A cache for query results. global_context->setQueryCache(config()); @@ -1611,7 +1573,7 @@ try /// context is destroyed. /// In addition this object has to be created before the loading of the tables. std::unique_ptr dns_cache_updater; - if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache")) + if (server_settings.disable_internal_dns_cache) { /// Disable DNS caching at all DNSResolver::instance().setDisableCacheFlag(); @@ -1621,7 +1583,7 @@ try { /// Initialize a watcher periodically updating DNS cache dns_cache_updater = std::make_unique( - global_context, config().getInt("dns_cache_update_period", 15), config().getUInt("dns_max_consecutive_failures", 5)); + global_context, server_settings.dns_cache_update_period, server_settings.dns_max_consecutive_failures); } if (dns_cache_updater) @@ -1886,7 +1848,7 @@ try LOG_INFO(log, "Closed all listening sockets."); /// Killing remaining queries. - if (!config().getBool("shutdown_wait_unfinished_queries", false)) + if (server_settings.shutdown_wait_unfinished_queries) global_context->getProcessList().killAllQueries(); if (current_connections) diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp index df96d83316c..1476a66c892 100644 --- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp @@ -102,19 +102,21 @@ public: if (!left_argument_constant_node && !right_argument_constant_node) return; - /** If we extract negative constant, aggregate function name must be updated. + /** Need reverse max <-> min for: * - * Example: SELECT min(-1 * id); - * Result: SELECT -1 * max(id); + * max(-1*value) -> -1*min(value) + * max(value/-2) -> min(value)/-2 + * max(1-value) -> 1-min(value) */ - std::string aggregate_function_name_if_constant_is_negative; - if (arithmetic_function_name == "multiply" || arithmetic_function_name == "divide") + auto get_reverse_aggregate_function_name = [](const std::string & aggregate_function_name) -> std::string { - if (lower_aggregate_function_name == "min") - aggregate_function_name_if_constant_is_negative = "max"; - else if (lower_aggregate_function_name == "max") - aggregate_function_name_if_constant_is_negative = "min"; - } + if (aggregate_function_name == "min") + return "max"; + else if (aggregate_function_name == "max") + return "min"; + else + return aggregate_function_name; + }; size_t arithmetic_function_argument_index = 0; @@ -126,11 +128,11 @@ public: /// Rewrite `aggregate_function(inner_function(constant, argument))` into `inner_function(constant, aggregate_function(argument))` const auto & left_argument_constant_value_literal = left_argument_constant_node->getValue(); - if (!aggregate_function_name_if_constant_is_negative.empty() && - left_argument_constant_value_literal < zeroField(left_argument_constant_value_literal)) - { - lower_aggregate_function_name = aggregate_function_name_if_constant_is_negative; - } + bool need_reverse = (arithmetic_function_name == "multiply" && left_argument_constant_value_literal < zeroField(left_argument_constant_value_literal)) + || (arithmetic_function_name == "minus"); + + if (need_reverse) + lower_aggregate_function_name = get_reverse_aggregate_function_name(lower_aggregate_function_name); arithmetic_function_argument_index = 1; } @@ -138,11 +140,10 @@ public: { /// Rewrite `aggregate_function(inner_function(argument, constant))` into `inner_function(aggregate_function(argument), constant)` const auto & right_argument_constant_value_literal = right_argument_constant_node->getValue(); - if (!aggregate_function_name_if_constant_is_negative.empty() && - right_argument_constant_value_literal < zeroField(right_argument_constant_value_literal)) - { - lower_aggregate_function_name = aggregate_function_name_if_constant_is_negative; - } + bool need_reverse = (arithmetic_function_name == "multiply" || arithmetic_function_name == "divide") && right_argument_constant_value_literal < zeroField(right_argument_constant_value_literal); + + if (need_reverse) + lower_aggregate_function_name = get_reverse_aggregate_function_name(lower_aggregate_function_name); arithmetic_function_argument_index = 0; } diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 05b643aa6af..c7b9f9aae08 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -233,11 +233,43 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q auto select_settings = select_query_typed.settings(); SettingsChanges settings_changes; + /// We are going to remove settings LIMIT and OFFSET and + /// further replace them with corresponding expression nodes + UInt64 limit = 0; + UInt64 offset = 0; + + /// Remove global settings limit and offset + if (const auto & settings_ref = updated_context->getSettingsRef(); settings_ref.limit || settings_ref.offset) + { + Settings settings = updated_context->getSettings(); + limit = settings.limit; + offset = settings.offset; + settings.limit = 0; + settings.offset = 0; + updated_context->setSettings(settings); + } + if (select_settings) { auto & set_query = select_settings->as(); - updated_context->applySettingsChanges(set_query.changes); - settings_changes = set_query.changes; + + /// Remove expression settings limit and offset + if (auto * limit_field = set_query.changes.tryGet("limit")) + { + limit = limit_field->safeGet(); + set_query.changes.removeSetting("limit"); + } + if (auto * offset_field = set_query.changes.tryGet("offset")) + { + offset = offset_field->safeGet(); + set_query.changes.removeSetting("offset"); + } + + if (!set_query.changes.empty()) + { + updated_context->applySettingsChanges(set_query.changes); + settings_changes = set_query.changes; + } } auto current_query_tree = std::make_shared(std::move(updated_context), std::move(settings_changes)); @@ -323,12 +355,32 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q if (select_limit_by) current_query_tree->getLimitByNode() = buildExpressionList(select_limit_by, current_context); + /// Combine limit expression with limit setting auto select_limit = select_query_typed.limitLength(); - if (select_limit) + if (select_limit && limit) + { + auto function_node = std::make_shared("least"); + function_node->getArguments().getNodes().push_back(buildExpression(select_limit, current_context)); + function_node->getArguments().getNodes().push_back(std::make_shared(limit)); + current_query_tree->getLimit() = std::move(function_node); + } + else if (limit) + current_query_tree->getLimit() = std::make_shared(limit); + else if (select_limit) current_query_tree->getLimit() = buildExpression(select_limit, current_context); + /// Combine offset expression with offset setting auto select_offset = select_query_typed.limitOffset(); - if (select_offset) + if (select_offset && offset) + { + auto function_node = std::make_shared("plus"); + function_node->getArguments().getNodes().push_back(buildExpression(select_offset, current_context)); + function_node->getArguments().getNodes().push_back(std::make_shared(offset)); + current_query_tree->getOffset() = std::move(function_node); + } + else if (offset) + current_query_tree->getOffset() = std::make_shared(offset); + else if (select_offset) current_query_tree->getOffset() = buildExpression(select_offset, current_context); return current_query_tree; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aef7dc6a38e..6c5142813c5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -577,8 +577,8 @@ if (TARGET ch_contrib::annoy) endif() if (TARGET ch_rust::skim) - # Add only -I, library is needed only for clickhouse-client/clickhouse-local dbms_target_include_directories(PRIVATE $) + dbms_target_link_libraries(PUBLIC ch_rust::skim) endif() include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake") diff --git a/src/Common/FieldVisitorToString.cpp b/src/Common/FieldVisitorToString.cpp index e0ec6b7a335..60834afab35 100644 --- a/src/Common/FieldVisitorToString.cpp +++ b/src/Common/FieldVisitorToString.cpp @@ -117,14 +117,14 @@ String FieldVisitorToString::operator() (const Map & x) const { WriteBufferFromOwnString wb; - wb << '('; + wb << '['; for (auto it = x.begin(); it != x.end(); ++it) { if (it != x.begin()) wb << ", "; wb << applyVisitor(*this, *it); } - wb << ')'; + wb << ']'; return wb.str(); } diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index 0c6e32cb383..d8ed1e205c8 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -100,8 +100,8 @@ private: bool required_substring_is_prefix; bool is_case_insensitive; std::string required_substring; - std::optional> case_sensitive_substring_searcher; - std::optional> case_insensitive_substring_searcher; + std::optional case_sensitive_substring_searcher; + std::optional case_insensitive_substring_searcher; std::unique_ptr re2; unsigned number_of_subpatterns; diff --git a/src/Common/SettingsChanges.cpp b/src/Common/SettingsChanges.cpp index 9fb4f361e09..7790c272606 100644 --- a/src/Common/SettingsChanges.cpp +++ b/src/Common/SettingsChanges.cpp @@ -46,4 +46,30 @@ Field * SettingsChanges::tryGet(std::string_view name) return &change->value; } +bool SettingsChanges::insertSetting(std::string_view name, const Field & value) +{ + auto it = std::find_if(begin(), end(), [&name](const SettingChange & change) { return change.name == name; }); + if (it != end()) + return false; + emplace_back(name, value); + return true; +} + +void SettingsChanges::setSetting(std::string_view name, const Field & value) +{ + if (auto * setting_value = tryGet(name)) + *setting_value = value; + else + insertSetting(name, value); +} + +bool SettingsChanges::removeSetting(std::string_view name) +{ + auto it = std::find_if(begin(), end(), [&name](const SettingChange & change) { return change.name == name; }); + if (it == end()) + return false; + erase(it); + return true; +} + } diff --git a/src/Common/SettingsChanges.h b/src/Common/SettingsChanges.h index 776dacb93e8..514e9a78911 100644 --- a/src/Common/SettingsChanges.h +++ b/src/Common/SettingsChanges.h @@ -28,6 +28,13 @@ public: bool tryGet(std::string_view name, Field & out_value) const; const Field * tryGet(std::string_view name) const; Field * tryGet(std::string_view name); + + /// Inserts element if doesn't exists and returns true, otherwise just returns false + bool insertSetting(std::string_view name, const Field & value); + /// Sets element to value, inserts if doesn't exist + void setSetting(std::string_view name, const Field & value); + /// If element exists - removes it and returns true, otherwise returns false + bool removeSetting(std::string_view name); }; } diff --git a/src/Common/StringSearcher.h b/src/Common/StringSearcher.h index a80c8c73292..ae440f9151b 100644 --- a/src/Common/StringSearcher.h +++ b/src/Common/StringSearcher.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -32,27 +31,476 @@ namespace ErrorCodes * In most cases, performance is less than Volnitsky (see Volnitsky.h). */ +namespace impl +{ class StringSearcherBase { public: bool force_fallback = false; + #ifdef __SSE2__ protected: - static constexpr auto n = sizeof(__m128i); - const Int64 page_size = ::getPageSize(); + static constexpr size_t N = sizeof(__m128i); - bool pageSafe(const void * const ptr) const + bool isPageSafe(const void * const ptr) const { - return ((page_size - 1) & reinterpret_cast(ptr)) <= page_size - n; + return ((page_size - 1) & reinterpret_cast(ptr)) <= page_size - N; } + +private: + const Int64 page_size = ::getPageSize(); #endif }; -/// Performs case-sensitive and case-insensitive search of UTF-8 strings +/// Performs case-sensitive or case-insensitive search of ASCII or UTF-8 strings template class StringSearcher; + +/// Case-sensitive ASCII and UTF8 searcher +template +class StringSearcher : public StringSearcherBase +{ +private: + /// string to be searched for + const uint8_t * const needle; + const uint8_t * const needle_end; + /// first character in `needle` + uint8_t first_needle_character = 0; + +#ifdef __SSE4_1__ + /// second character of "needle" (if its length is > 1) + uint8_t second_needle_character = 0; + /// first/second needle character broadcasted into a 16 bytes vector + __m128i first_needle_character_vec; + __m128i second_needle_character_vec; + /// vector of first 16 characters of `needle` + __m128i cache = _mm_setzero_si128(); + uint16_t cachemask = 0; +#endif + +public: + template + requires (sizeof(CharT) == 1) + StringSearcher(const CharT * needle_, size_t needle_size) + : needle(reinterpret_cast(needle_)) + , needle_end(needle + needle_size) + { + if (needle_size == 0) + return; + + first_needle_character = *needle; + +#ifdef __SSE4_1__ + first_needle_character_vec = _mm_set1_epi8(first_needle_character); + if (needle_size > 1) + { + second_needle_character = *(needle + 1); + second_needle_character_vec = _mm_set1_epi8(second_needle_character); + } + const auto * needle_pos = needle; + + for (uint8_t i = 0; i < N; ++i) + { + cache = _mm_srli_si128(cache, 1); + + if (needle_pos != needle_end) + { + cache = _mm_insert_epi8(cache, *needle_pos, N - 1); + cachemask |= 1 << i; + ++needle_pos; + } + } +#endif + } + + template + requires (sizeof(CharT) == 1) + ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * /*haystack_end*/, const CharT * pos) const + { +#ifdef __SSE4_1__ + if (isPageSafe(pos)) + { + const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast(pos)); + const __m128i comparison_result = _mm_cmpeq_epi8(haystack_characters, cache); + const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result); + + if (0xffff == cachemask) + { + if (comparison_result_mask == cachemask) + { + pos += N; + const auto * needle_pos = needle + N; + + while (needle_pos < needle_end && *pos == *needle_pos) + ++pos, ++needle_pos; + + if (needle_pos == needle_end) + return true; + } + } + else if ((comparison_result_mask & cachemask) == cachemask) + return true; + + return false; + } +#endif + + if (*pos == first_needle_character) + { + ++pos; + const auto * needle_pos = needle + 1; + + while (needle_pos < needle_end && *pos == *needle_pos) + ++pos, ++needle_pos; + + if (needle_pos == needle_end) + return true; + } + + return false; + } + + template + requires (sizeof(CharT) == 1) + const CharT * search(const CharT * haystack, const CharT * const haystack_end) const + { + const auto needle_size = needle_end - needle; + + if (needle == needle_end) + return haystack; + +#ifdef __SSE4_1__ + /// Fast path for single-character needles. Compare 16 characters of the haystack against the needle character at once. + if (needle_size == 1) + { + while (haystack < haystack_end) + { + if (haystack + N <= haystack_end && isPageSafe(haystack)) + { + const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast(haystack)); + const __m128i comparison_result = _mm_cmpeq_epi8(haystack_characters, first_needle_character_vec); + const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result); + if (comparison_result_mask == 0) + { + haystack += N; + continue; + } + + const int offset = std::countr_zero(comparison_result_mask); + haystack += offset; + + return haystack; + } + + if (haystack == haystack_end) + return haystack_end; + + if (*haystack == first_needle_character) + return haystack; + + ++haystack; + } + + return haystack_end; + } +#endif + + while (haystack < haystack_end && haystack_end - haystack >= needle_size) + { +#ifdef __SSE4_1__ + /// Compare the [0:15] bytes from haystack and broadcasted 16 bytes vector from first character of needle. + /// Compare the [1:16] bytes from haystack and broadcasted 16 bytes vector from second character of needle. + /// Bit AND the results of above two comparisons and get the mask. + if ((haystack + 1 + N) <= haystack_end && isPageSafe(haystack + 1)) + { + const __m128i haystack_characters_from_1st = _mm_loadu_si128(reinterpret_cast(haystack)); + const __m128i haystack_characters_from_2nd = _mm_loadu_si128(reinterpret_cast(haystack + 1)); + const __m128i comparison_result_1st = _mm_cmpeq_epi8(haystack_characters_from_1st, first_needle_character_vec); + const __m128i comparison_result_2nd = _mm_cmpeq_epi8(haystack_characters_from_2nd, second_needle_character_vec); + const __m128i comparison_result_combined = _mm_and_si128(comparison_result_1st, comparison_result_2nd); + const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result_combined); + /// If the mask = 0, then first two characters [0:1] from needle are not in the [0:17] bytes of haystack. + if (comparison_result_mask == 0) + { + haystack += N; + continue; + } + + const int offset = std::countr_zero(comparison_result_mask); + haystack += offset; + + if (haystack + N <= haystack_end && isPageSafe(haystack)) + { + /// Already find the haystack position where the [pos:pos + 1] two characters exactly match the first two characters of needle. + /// Compare the 16 bytes from needle (cache) and the first 16 bytes from haystack at once if the haystack size >= 16 bytes. + const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast(haystack)); + const __m128i comparison_result_cache = _mm_cmpeq_epi8(haystack_characters, cache); + const uint16_t mask_offset = _mm_movemask_epi8(comparison_result_cache); + + if (0xffff == cachemask) + { + if (mask_offset == cachemask) + { + const auto * haystack_pos = haystack + N; + const auto * needle_pos = needle + N; + + while (haystack_pos < haystack_end && needle_pos < needle_end && + *haystack_pos == *needle_pos) + ++haystack_pos, ++needle_pos; + + if (needle_pos == needle_end) + return haystack; + } + } + else if ((mask_offset & cachemask) == cachemask) + return haystack; + + ++haystack; + continue; + } + } +#endif + + if (haystack == haystack_end) + return haystack_end; + + if (*haystack == first_needle_character) + { + const auto * haystack_pos = haystack + 1; + const auto * needle_pos = needle + 1; + + while (haystack_pos < haystack_end && needle_pos < needle_end && + *haystack_pos == *needle_pos) + ++haystack_pos, ++needle_pos; + + if (needle_pos == needle_end) + return haystack; + } + + ++haystack; + } + + return haystack_end; + } + + template + requires (sizeof(CharT) == 1) + const CharT * search(const CharT * haystack, size_t haystack_size) const + { + return search(haystack, haystack + haystack_size); + } +}; + + +/// Case-insensitive ASCII searcher +template <> +class StringSearcher : public StringSearcherBase +{ +private: + /// string to be searched for + const uint8_t * const needle; + const uint8_t * const needle_end; + /// lower and uppercase variants of the first character in `needle` + uint8_t l = 0; + uint8_t u = 0; + +#ifdef __SSE4_1__ + /// vectors filled with `l` and `u`, for determining leftmost position of the first symbol + __m128i patl, patu; + /// lower and uppercase vectors of first 16 characters of `needle` + __m128i cachel = _mm_setzero_si128(), cacheu = _mm_setzero_si128(); + int cachemask = 0; +#endif + +public: + template + requires (sizeof(CharT) == 1) + StringSearcher(const CharT * needle_, size_t needle_size) + : needle(reinterpret_cast(needle_)) + , needle_end(needle + needle_size) + { + if (needle_size == 0) + return; + + l = static_cast(std::tolower(*needle)); + u = static_cast(std::toupper(*needle)); + +#ifdef __SSE4_1__ + patl = _mm_set1_epi8(l); + patu = _mm_set1_epi8(u); + + const auto * needle_pos = needle; + + for (size_t i = 0; i < N; ++i) + { + cachel = _mm_srli_si128(cachel, 1); + cacheu = _mm_srli_si128(cacheu, 1); + + if (needle_pos != needle_end) + { + cachel = _mm_insert_epi8(cachel, std::tolower(*needle_pos), N - 1); + cacheu = _mm_insert_epi8(cacheu, std::toupper(*needle_pos), N - 1); + cachemask |= 1 << i; + ++needle_pos; + } + } +#endif + } + + template + requires (sizeof(CharT) == 1) + ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * /*haystack_end*/, const CharT * pos) const + { +#ifdef __SSE4_1__ + if (isPageSafe(pos)) + { + const auto v_haystack = _mm_loadu_si128(reinterpret_cast(pos)); + const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel); + const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu); + const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u); + const auto mask = _mm_movemask_epi8(v_against_l_or_u); + + if (0xffff == cachemask) + { + if (mask == cachemask) + { + pos += N; + const auto * needle_pos = needle + N; + + while (needle_pos < needle_end && std::tolower(*pos) == std::tolower(*needle_pos)) + { + ++pos; + ++needle_pos; + } + + if (needle_pos == needle_end) + return true; + } + } + else if ((mask & cachemask) == cachemask) + return true; + + return false; + } +#endif + + if (*pos == l || *pos == u) + { + ++pos; + const auto * needle_pos = needle + 1; + + while (needle_pos < needle_end && std::tolower(*pos) == std::tolower(*needle_pos)) + { + ++pos; + ++needle_pos; + } + + if (needle_pos == needle_end) + return true; + } + + return false; + } + + template + requires (sizeof(CharT) == 1) + const CharT * search(const CharT * haystack, const CharT * const haystack_end) const + { + if (needle == needle_end) + return haystack; + + while (haystack < haystack_end) + { +#ifdef __SSE4_1__ + if (haystack + N <= haystack_end && isPageSafe(haystack)) + { + const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); + const auto v_against_l = _mm_cmpeq_epi8(v_haystack, patl); + const auto v_against_u = _mm_cmpeq_epi8(v_haystack, patu); + const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u); + + const auto mask = _mm_movemask_epi8(v_against_l_or_u); + + if (mask == 0) + { + haystack += N; + continue; + } + + const auto offset = __builtin_ctz(mask); + haystack += offset; + + if (haystack + N <= haystack_end && isPageSafe(haystack)) + { + const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast(haystack)); + const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel); + const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu); + const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset); + const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset); + + if (0xffff == cachemask) + { + if (mask_offset == cachemask) + { + const auto * haystack_pos = haystack + N; + const auto * needle_pos = needle + N; + + while (haystack_pos < haystack_end && needle_pos < needle_end && + std::tolower(*haystack_pos) == std::tolower(*needle_pos)) + { + ++haystack_pos; + ++needle_pos; + } + + if (needle_pos == needle_end) + return haystack; + } + } + else if ((mask_offset & cachemask) == cachemask) + return haystack; + + ++haystack; + continue; + } + } +#endif + + if (haystack == haystack_end) + return haystack_end; + + if (*haystack == l || *haystack == u) + { + const auto * haystack_pos = haystack + 1; + const auto * needle_pos = needle + 1; + + while (haystack_pos < haystack_end && needle_pos < needle_end && + std::tolower(*haystack_pos) == std::tolower(*needle_pos)) + { + ++haystack_pos; + ++needle_pos; + } + + if (needle_pos == needle_end) + return haystack; + } + + ++haystack; + } + + return haystack_end; + } + + template + requires (sizeof(CharT) == 1) + const CharT * search(const CharT * haystack, size_t haystack_size) const + { + return search(haystack, haystack + haystack_size); + } +}; + + /// Case-insensitive UTF-8 searcher template <> class StringSearcher : public StringSearcherBase @@ -65,9 +513,9 @@ private: const size_t needle_size; const uint8_t * const needle_end = needle + needle_size; /// lower and uppercase variants of the first octet of the first character in `needle` - bool first_needle_symbol_is_ascii{}; - uint8_t l{}; - uint8_t u{}; + bool first_needle_symbol_is_ascii = false; + uint8_t l = 0; + uint8_t u = 0; #ifdef __SSE4_1__ /// vectors filled with `l` and `u`, for determining leftmost position of the first symbol @@ -76,18 +524,19 @@ private: /// lower and uppercase vectors of first 16 characters of `needle` __m128i cachel = _mm_setzero_si128(); __m128i cacheu = _mm_setzero_si128(); - int cachemask{}; - size_t cache_valid_len{}; - size_t cache_actual_len{}; + int cachemask = 0; + size_t cache_valid_len = 0; + size_t cache_actual_len = 0; #endif public: template requires (sizeof(CharT) == 1) - StringSearcher(const CharT * needle_, const size_t needle_size_) - : needle{reinterpret_cast(needle_)}, needle_size{needle_size_} + StringSearcher(const CharT * needle_, size_t needle_size_) + : needle(reinterpret_cast(needle_)) + , needle_size(needle_size_) { - if (0 == needle_size) + if (needle_size == 0) return; UTF8SequenceBuffer l_seq; @@ -140,7 +589,7 @@ public: const auto * needle_pos = needle; - for (size_t i = 0; i < n;) + for (size_t i = 0; i < N;) { if (needle_pos == needle_end) { @@ -171,18 +620,18 @@ public: } cache_actual_len += src_len; - if (cache_actual_len < n) + if (cache_actual_len < N) cache_valid_len += src_len; - for (size_t j = 0; j < src_len && i < n; ++j, ++i) + for (size_t j = 0; j < src_len && i < N; ++j, ++i) { cachel = _mm_srli_si128(cachel, 1); cacheu = _mm_srli_si128(cacheu, 1); if (needle_pos != needle_end) { - cachel = _mm_insert_epi8(cachel, l_seq[j], n - 1); - cacheu = _mm_insert_epi8(cacheu, u_seq[j], n - 1); + cachel = _mm_insert_epi8(cachel, l_seq[j], N - 1); + cacheu = _mm_insert_epi8(cacheu, u_seq[j], N - 1); cachemask |= 1 << i; ++needle_pos; @@ -225,7 +674,7 @@ public: { #ifdef __SSE4_1__ - if (pageSafe(pos) && !force_fallback) + if (isPageSafe(pos) && !force_fallback) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(pos)); const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel); @@ -269,13 +718,13 @@ public: requires (sizeof(CharT) == 1) const CharT * search(const CharT * haystack, const CharT * const haystack_end) const { - if (0 == needle_size) + if (needle_size == 0) return haystack; while (haystack < haystack_end) { #ifdef __SSE4_1__ - if (haystack + n <= haystack_end && pageSafe(haystack) && !force_fallback) + if (haystack + N <= haystack_end && isPageSafe(haystack) && !force_fallback) { const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); const auto v_against_l = _mm_cmpeq_epi8(v_haystack, patl); @@ -286,7 +735,7 @@ public: if (mask == 0) { - haystack += n; + haystack += N; UTF8::syncForward(haystack, haystack_end); continue; } @@ -294,7 +743,7 @@ public: const auto offset = __builtin_ctz(mask); haystack += offset; - if (haystack + n <= haystack_end && pageSafe(haystack)) + if (haystack + N <= haystack_end && isPageSafe(haystack)) { const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast(haystack)); const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel); @@ -344,453 +793,13 @@ public: template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const + const CharT * search(const CharT * haystack, size_t haystack_size) const { return search(haystack, haystack + haystack_size); } }; -/// Case-insensitive ASCII searcher -template <> -class StringSearcher : public StringSearcherBase -{ -private: - /// string to be searched for - const uint8_t * const needle; - const uint8_t * const needle_end; - /// lower and uppercase variants of the first character in `needle` - uint8_t l{}; - uint8_t u{}; - -#ifdef __SSE4_1__ - /// vectors filled with `l` and `u`, for determining leftmost position of the first symbol - __m128i patl, patu; - /// lower and uppercase vectors of first 16 characters of `needle` - __m128i cachel = _mm_setzero_si128(), cacheu = _mm_setzero_si128(); - int cachemask{}; -#endif - -public: - template - requires (sizeof(CharT) == 1) - StringSearcher(const CharT * needle_, const size_t needle_size) - : needle{reinterpret_cast(needle_)}, needle_end{needle + needle_size} - { - if (0 == needle_size) - return; - - l = static_cast(std::tolower(*needle)); - u = static_cast(std::toupper(*needle)); - -#ifdef __SSE4_1__ - patl = _mm_set1_epi8(l); - patu = _mm_set1_epi8(u); - - const auto * needle_pos = needle; - - for (const auto i : collections::range(0, n)) - { - cachel = _mm_srli_si128(cachel, 1); - cacheu = _mm_srli_si128(cacheu, 1); - - if (needle_pos != needle_end) - { - cachel = _mm_insert_epi8(cachel, std::tolower(*needle_pos), n - 1); - cacheu = _mm_insert_epi8(cacheu, std::toupper(*needle_pos), n - 1); - cachemask |= 1 << i; - ++needle_pos; - } - } -#endif - } - - template - requires (sizeof(CharT) == 1) - ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * /*haystack_end*/, const CharT * pos) const - { -#ifdef __SSE4_1__ - if (pageSafe(pos)) - { - const auto v_haystack = _mm_loadu_si128(reinterpret_cast(pos)); - const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel); - const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu); - const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u); - const auto mask = _mm_movemask_epi8(v_against_l_or_u); - - if (0xffff == cachemask) - { - if (mask == cachemask) - { - pos += n; - const auto * needle_pos = needle + n; - - while (needle_pos < needle_end && std::tolower(*pos) == std::tolower(*needle_pos)) - { - ++pos; - ++needle_pos; - } - - if (needle_pos == needle_end) - return true; - } - } - else if ((mask & cachemask) == cachemask) - return true; - - return false; - } -#endif - - if (*pos == l || *pos == u) - { - ++pos; - const auto * needle_pos = needle + 1; - - while (needle_pos < needle_end && std::tolower(*pos) == std::tolower(*needle_pos)) - { - ++pos; - ++needle_pos; - } - - if (needle_pos == needle_end) - return true; - } - - return false; - } - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const CharT * const haystack_end) const - { - if (needle == needle_end) - return haystack; - - while (haystack < haystack_end) - { -#ifdef __SSE4_1__ - if (haystack + n <= haystack_end && pageSafe(haystack)) - { - const auto v_haystack = _mm_loadu_si128(reinterpret_cast(haystack)); - const auto v_against_l = _mm_cmpeq_epi8(v_haystack, patl); - const auto v_against_u = _mm_cmpeq_epi8(v_haystack, patu); - const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u); - - const auto mask = _mm_movemask_epi8(v_against_l_or_u); - - if (mask == 0) - { - haystack += n; - continue; - } - - const auto offset = __builtin_ctz(mask); - haystack += offset; - - if (haystack + n <= haystack_end && pageSafe(haystack)) - { - const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast(haystack)); - const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel); - const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu); - const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset); - const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset); - - if (0xffff == cachemask) - { - if (mask_offset == cachemask) - { - const auto * haystack_pos = haystack + n; - const auto * needle_pos = needle + n; - - while (haystack_pos < haystack_end && needle_pos < needle_end && - std::tolower(*haystack_pos) == std::tolower(*needle_pos)) - { - ++haystack_pos; - ++needle_pos; - } - - if (needle_pos == needle_end) - return haystack; - } - } - else if ((mask_offset & cachemask) == cachemask) - return haystack; - - ++haystack; - continue; - } - } -#endif - - if (haystack == haystack_end) - return haystack_end; - - if (*haystack == l || *haystack == u) - { - const auto * haystack_pos = haystack + 1; - const auto * needle_pos = needle + 1; - - while (haystack_pos < haystack_end && needle_pos < needle_end && - std::tolower(*haystack_pos) == std::tolower(*needle_pos)) - { - ++haystack_pos; - ++needle_pos; - } - - if (needle_pos == needle_end) - return haystack; - } - - ++haystack; - } - - return haystack_end; - } - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const - { - return search(haystack, haystack + haystack_size); - } -}; - - -/// Case-sensitive searcher (both ASCII and UTF-8) -template -class StringSearcher : public StringSearcherBase -{ -private: - /// string to be searched for - const uint8_t * const needle; - const uint8_t * const needle_end; - /// first character in `needle` - uint8_t first_needle_character = 0; - -#ifdef __SSE4_1__ - /// second character of "needle" (if its length is > 1) - uint8_t second_needle_character = 0; - /// The first/second needle character broadcasted into a 16 bytes vector - __m128i first_needle_character_vec; - __m128i second_needle_character_vec; - /// vector of first 16 characters of `needle` - __m128i cache = _mm_setzero_si128(); - uint16_t cachemask = 0; -#endif - -public: - template - requires (sizeof(CharT) == 1) - StringSearcher(const CharT * needle_, const size_t needle_size) - : needle{reinterpret_cast(needle_)}, needle_end{needle + needle_size} - { - if (0 == needle_size) - return; - - first_needle_character = *needle; - -#ifdef __SSE4_1__ - first_needle_character_vec = _mm_set1_epi8(first_needle_character); - if (needle_size > 1) - { - second_needle_character = *(needle + 1); - second_needle_character_vec = _mm_set1_epi8(second_needle_character); - } - const auto * needle_pos = needle; - - for (uint8_t i = 0; i < n; ++i) - { - cache = _mm_srli_si128(cache, 1); - - if (needle_pos != needle_end) - { - cache = _mm_insert_epi8(cache, *needle_pos, n - 1); - cachemask |= 1 << i; - ++needle_pos; - } - } -#endif - } - - template - requires (sizeof(CharT) == 1) - ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * /*haystack_end*/, const CharT * pos) const - { -#ifdef __SSE4_1__ - if (pageSafe(pos)) - { - const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast(pos)); - const __m128i comparison_result = _mm_cmpeq_epi8(haystack_characters, cache); - const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result); - - if (0xffff == cachemask) - { - if (comparison_result_mask == cachemask) - { - pos += n; - const auto * needle_pos = needle + n; - - while (needle_pos < needle_end && *pos == *needle_pos) - ++pos, ++needle_pos; - - if (needle_pos == needle_end) - return true; - } - } - else if ((comparison_result_mask & cachemask) == cachemask) - return true; - - return false; - } -#endif - - if (*pos == first_needle_character) - { - ++pos; - const auto * needle_pos = needle + 1; - - while (needle_pos < needle_end && *pos == *needle_pos) - ++pos, ++needle_pos; - - if (needle_pos == needle_end) - return true; - } - - return false; - } - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const CharT * const haystack_end) const - { - const auto needle_size = needle_end - needle; - - if (needle == needle_end) - return haystack; - -#ifdef __SSE4_1__ - /// Fast path for single-character needles. Compare 16 characters of the haystack against the needle character at once. - if (needle_size == 1) - { - while (haystack < haystack_end) - { - if (haystack + n <= haystack_end && pageSafe(haystack)) - { - const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast(haystack)); - const __m128i comparison_result = _mm_cmpeq_epi8(haystack_characters, first_needle_character_vec); - const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result); - if (comparison_result_mask == 0) - { - haystack += n; - continue; - } - - const auto offset = std::countr_zero(comparison_result_mask); - haystack += offset; - - return haystack; - } - - if (haystack == haystack_end) - return haystack_end; - - if (*haystack == first_needle_character) - return haystack; - - ++haystack; - } - - return haystack_end; - } -#endif - - while (haystack < haystack_end && haystack_end - haystack >= needle_size) - { -#ifdef __SSE4_1__ - /// Compare the [0:15] bytes from haystack and broadcasted 16 bytes vector from first character of needle. - /// Compare the [1:16] bytes from haystack and broadcasted 16 bytes vector from second character of needle. - /// Bit AND the results of above two comparisons and get the mask. - if ((haystack + 1 + n) <= haystack_end && pageSafe(haystack + 1)) - { - const __m128i haystack_characters_from_1st = _mm_loadu_si128(reinterpret_cast(haystack)); - const __m128i haystack_characters_from_2nd = _mm_loadu_si128(reinterpret_cast(haystack + 1)); - const __m128i comparison_result_1st = _mm_cmpeq_epi8(haystack_characters_from_1st, first_needle_character_vec); - const __m128i comparison_result_2nd = _mm_cmpeq_epi8(haystack_characters_from_2nd, second_needle_character_vec); - const __m128i comparison_result_combined = _mm_and_si128(comparison_result_1st, comparison_result_2nd); - const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result_combined); - /// If the mask = 0, then first two characters [0:1] from needle are not in the [0:17] bytes of haystack. - if (comparison_result_mask == 0) - { - haystack += n; - continue; - } - - const auto offset = std::countr_zero(comparison_result_mask); - haystack += offset; - - if (haystack + n <= haystack_end && pageSafe(haystack)) - { - /// Already find the haystack position where the [pos:pos + 1] two characters exactly match the first two characters of needle. - /// Compare the 16 bytes from needle (cache) and the first 16 bytes from haystack at once if the haystack size >= 16 bytes. - const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast(haystack)); - const __m128i comparison_result_cache = _mm_cmpeq_epi8(haystack_characters, cache); - const uint16_t mask_offset = _mm_movemask_epi8(comparison_result_cache); - - if (0xffff == cachemask) - { - if (mask_offset == cachemask) - { - const auto * haystack_pos = haystack + n; - const auto * needle_pos = needle + n; - - while (haystack_pos < haystack_end && needle_pos < needle_end && - *haystack_pos == *needle_pos) - ++haystack_pos, ++needle_pos; - - if (needle_pos == needle_end) - return haystack; - } - } - else if ((mask_offset & cachemask) == cachemask) - return haystack; - - ++haystack; - continue; - } - } -#endif - - if (haystack == haystack_end) - return haystack_end; - - if (*haystack == first_needle_character) - { - const auto * haystack_pos = haystack + 1; - const auto * needle_pos = needle + 1; - - while (haystack_pos < haystack_end && needle_pos < needle_end && - *haystack_pos == *needle_pos) - ++haystack_pos, ++needle_pos; - - if (needle_pos == needle_end) - return haystack; - } - - ++haystack; - } - - return haystack_end; - } - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const - { - return search(haystack, haystack + haystack_size); - } -}; - // Searches for needle surrounded by token-separators. // Separators are anything inside ASCII (0-128) and not alphanum. // Any value outside of basic ASCII (>=128) is considered a non-separator symbol, hence UTF-8 strings @@ -804,14 +813,12 @@ class TokenSearcher : public StringSearcherBase public: template requires (sizeof(CharT) == 1) - TokenSearcher(const CharT * needle_, const size_t needle_size_) - : searcher{needle_, needle_size_}, - needle_size(needle_size_) + TokenSearcher(const CharT * needle_, size_t needle_size_) + : searcher(needle_, needle_size_) + , needle_size(needle_size_) { if (std::any_of(needle_, needle_ + needle_size_, isTokenSeparator)) - { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Needle must not contain whitespace or separator characters"); - } } @@ -848,7 +855,7 @@ public: template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const + const CharT * search(const CharT * haystack, size_t haystack_size) const { return search(haystack, haystack + haystack_size); } @@ -867,26 +874,27 @@ public: } }; +} -using ASCIICaseSensitiveStringSearcher = StringSearcher; -using ASCIICaseInsensitiveStringSearcher = StringSearcher; -using UTF8CaseSensitiveStringSearcher = StringSearcher; -using UTF8CaseInsensitiveStringSearcher = StringSearcher; -using ASCIICaseSensitiveTokenSearcher = TokenSearcher; -using ASCIICaseInsensitiveTokenSearcher = TokenSearcher; +using ASCIICaseSensitiveStringSearcher = impl::StringSearcher; +using ASCIICaseInsensitiveStringSearcher = impl::StringSearcher; +using UTF8CaseSensitiveStringSearcher = impl::StringSearcher; +using UTF8CaseInsensitiveStringSearcher = impl::StringSearcher; +using ASCIICaseSensitiveTokenSearcher = impl::TokenSearcher; +using ASCIICaseInsensitiveTokenSearcher = impl::TokenSearcher; /// Use only with short haystacks where cheap initialization is required. template -struct StdLibASCIIStringSearcher : public StringSearcherBase +struct StdLibASCIIStringSearcher { const char * const needle_start; const char * const needle_end; template requires (sizeof(CharT) == 1) - StdLibASCIIStringSearcher(const CharT * const needle_start_, const size_t needle_size_) - : needle_start{reinterpret_cast(needle_start_)} - , needle_end{reinterpret_cast(needle_start) + needle_size_} + StdLibASCIIStringSearcher(const CharT * const needle_start_, size_t needle_size_) + : needle_start(reinterpret_cast(needle_start_)) + , needle_end(reinterpret_cast(needle_start) + needle_size_) {} template @@ -894,22 +902,18 @@ struct StdLibASCIIStringSearcher : public StringSearcherBase const CharT * search(const CharT * haystack_start, const CharT * const haystack_end) const { if constexpr (CaseInsensitive) - { return std::search( haystack_start, haystack_end, needle_start, needle_end, [](char c1, char c2) {return std::toupper(c1) == std::toupper(c2);}); - } else - { return std::search( haystack_start, haystack_end, needle_start, needle_end, [](char c1, char c2) {return c1 == c2;}); - } } template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack_start, const size_t haystack_length) const + const CharT * search(const CharT * haystack_start, size_t haystack_length) const { return search(haystack_start, haystack_start + haystack_length); } diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index c49bdcf2739..521422b780e 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -107,7 +107,9 @@ public: public: const String & getName() const; Field getValue() const; + Field getDefaultValue() const; String getValueString() const; + String getDefaultValueString() const; bool isValueChanged() const; const char * getTypeName() const; const char * getDescription() const; @@ -797,6 +799,17 @@ Field BaseSettings::SettingFieldRef::getValue() const return accessor->getValue(*settings, index); } +template +Field BaseSettings::SettingFieldRef::getDefaultValue() const +{ + if constexpr (Traits::allow_custom_settings) + { + if (custom_setting) + return static_cast(custom_setting->second); + } + return accessor->getDefaultValue(index); +} + template String BaseSettings::SettingFieldRef::getValueString() const { @@ -808,6 +821,17 @@ String BaseSettings::SettingFieldRef::getValueString() const return accessor->getValueString(*settings, index); } +template +String BaseSettings::SettingFieldRef::getDefaultValueString() const +{ + if constexpr (Traits::allow_custom_settings) + { + if (custom_setting) + return custom_setting->second.toString(); + } + return accessor->getDefaultValueString(index); +} + template bool BaseSettings::SettingFieldRef::isValueChanged() const { @@ -902,7 +926,8 @@ using AliasMap = std::unordered_map; void resetValueToDefault(Data & data, size_t index) const { return field_infos[index].reset_value_to_default_function(data); } \ void writeBinary(const Data & data, size_t index, WriteBuffer & out) const { return field_infos[index].write_binary_function(data, out); } \ void readBinary(Data & data, size_t index, ReadBuffer & in) const { return field_infos[index].read_binary_function(data, in); } \ - \ + Field getDefaultValue(size_t index) const { return field_infos[index].get_default_value_function(); } \ + String getDefaultValueString(size_t index) const { return field_infos[index].get_default_value_string_function(); } \ private: \ Accessor(); \ struct FieldInfo \ @@ -923,6 +948,8 @@ using AliasMap = std::unordered_map; void (*reset_value_to_default_function)(Data &) ; \ void (*write_binary_function)(const Data &, WriteBuffer &) ; \ void (*read_binary_function)(Data &, ReadBuffer &) ; \ + Field (*get_default_value_function)() ; \ + String (*get_default_value_string_function)() ; \ }; \ std::vector field_infos; \ std::unordered_map name_to_index_map; \ @@ -1033,6 +1060,8 @@ struct DefineAliases [](const Data & data) -> bool { return data.NAME.changed; }, \ [](Data & data) { data.NAME = SettingField##TYPE{DEFAULT}; }, \ [](const Data & data, WriteBuffer & out) { data.NAME.writeBinary(out); }, \ - [](Data & data, ReadBuffer & in) { data.NAME.readBinary(in); } \ + [](Data & data, ReadBuffer & in) { data.NAME.readBinary(in); }, \ + []() -> Field { return static_cast(SettingField##TYPE{DEFAULT}); }, \ + []() -> String { return SettingField##TYPE{DEFAULT}.toString(); } \ }); } diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp new file mode 100644 index 00000000000..c50a67b04c9 --- /dev/null +++ b/src/Core/ServerSettings.cpp @@ -0,0 +1,19 @@ +#include "ServerSettings.h" +#include + +namespace DB +{ + +IMPLEMENT_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS) + +void ServerSettings::loadSettingsFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + for (auto setting : all()) + { + const auto & name = setting.getName(); + if (config.has(name)) + set(name, config.getString(name)); + } +} + +} diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h new file mode 100644 index 00000000000..1e884266c17 --- /dev/null +++ b/src/Core/ServerSettings.h @@ -0,0 +1,78 @@ +#pragma once + + +#include + + +namespace Poco::Util +{ +class AbstractConfiguration; +} + +namespace DB +{ + +#define SERVER_SETTINGS(M, ALIAS) \ + M(Bool, show_addresses_in_stack_traces, true, "If it is set true will show addresses in stack traces", 0) \ + M(Bool, shutdown_wait_unfinished_queries, false, "If set true ClickHouse will wait for running queries finish before shutdown.", 0) \ + M(UInt64, max_thread_pool_size, 10000, "The maximum number of threads that could be allocated from the OS and used for query execution and background operations.", 0) \ + M(UInt64, max_thread_pool_free_size, 1000, "The maximum number of threads that will always stay in a global thread pool once allocated and remain idle in case of insufficient number of tasks.", 0) \ + M(UInt64, thread_pool_queue_size, 10000, "The maximum number of tasks that will be placed in a queue and wait for execution.", 0) \ + M(UInt64, max_io_thread_pool_size, 100, "The maximum number of threads that would be used for IO operations", 0) \ + M(UInt64, max_io_thread_pool_free_size, 0, "Max free size for IO thread pool.", 0) \ + M(UInt64, io_thread_pool_queue_size, 10000, "Queue size for IO thread pool.", 0) \ + M(Int32, max_connections, 1024, "Max server connections.", 0) \ + M(UInt32, asynchronous_metrics_update_period_s, 1, "Period in seconds for updating asynchronous metrics.", 0) \ + M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating asynchronous metrics.", 0) \ + M(UInt32, max_threads_for_connection_collector, 10, "The maximum number of threads that will be used for draining connections asynchronously in a background upon finishing executing distributed queries.", 0) \ + M(String, default_database, "default", "Default database name.", 0) \ + M(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \ + M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting., ", 0) \ + M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \ + M(UInt64, max_server_memory_usage, 0, "Limit on total memory usage. Zero means Unlimited.", 0) \ + M(Double, max_server_memory_usage_to_ram_ratio, 0.9, "Same as max_server_memory_usage but in to ram ratio. Allows to lower max memory on low-memory systems.", 0) \ + M(Bool, allow_use_jemalloc_memory, true, "Allows to use jemalloc memory.", 0) \ + \ + M(UInt64, max_concurrent_queries, 0, "Limit on total number of concurrently executed queries. Zero means Unlimited.", 0) \ + M(UInt64, max_concurrent_insert_queries, 0, "Limit on total number of concurrently insert queries. Zero means Unlimited.", 0) \ + M(UInt64, max_concurrent_select_queries, 0, "Limit on total number of concurrently select queries. Zero means Unlimited.", 0) \ + \ + M(Double, cache_size_to_ram_max_ratio, 0.5, "Set cache size ro ram max ratio. Allows to lower cache size on low-memory systems.", 0) \ + M(String, uncompressed_cache_policy, "SLRU", "Uncompressed cache policy name.", 0) \ + M(UInt64, uncompressed_cache_size, 0, "Size of cache for uncompressed blocks. Zero means disabled.", 0) \ + M(UInt64, mark_cache_size, 5368709120, "Size of cache for marks (index of MergeTree family of tables).", 0) \ + M(String, mark_cache_policy, "SLRU", "Mark cache policy name.", 0) \ + M(UInt64, index_uncompressed_cache_size, 0, "Size of cache for uncompressed blocks of MergeTree indices. Zero means disabled.", 0) \ + M(UInt64, index_mark_cache_size, 0, "Size of cache for index marks. Zero means disabled.", 0) \ + M(UInt64, mmap_cache_size, 1000, "A cache for mmapped files.", 0) /* The choice of default is arbitrary. */ \ + \ + M(Bool, disable_internal_dns_cache, false, "Disable internal DNS caching at all.", 0) \ + M(Int32, dns_cache_update_period, 15, "Internal DNS cache update period in seconds.", 0) \ + M(UInt32, dns_max_consecutive_failures, 1024, "Max server connections.", 0) \ + \ + M(UInt64, max_table_size_to_drop, 50000000000lu, "If size of a table is greater than this value (in bytes) than table could not be dropped with any DROP query.", 0) \ + M(UInt64, max_partition_size_to_drop, 50000000000lu, "Same as max_table_size_to_drop, but for the partitions.", 0) \ + M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means Unlimited.", 0) \ + M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ + \ + M(UInt64, background_pool_size, 16, "The maximum number of threads what will be used for merging or mutating data parts for *MergeTree-engine tables in a background.", 0) \ + M(UInt64, background_merges_mutations_concurrency_ratio, 2, "The multiplier which shows the relation between the number of tasks that could be executed concurrently and the number of threads being used.", 0) \ + M(String, background_merges_mutations_scheduling_policy, "round_robin", "The policy on how to perform a scheduling for background merges and mutations. Possible values are: `round_robin` and `shortest_task_first`. ", 0) \ + M(UInt64, background_move_pool_size, 8, "The maximum number of threads that will be used for moving data parts to another disk or volume for *MergeTree-engine tables in a background.", 0) \ + M(UInt64, background_fetches_pool_size, 8, "The maximum number of threads that will be used for fetching data parts from another replica for *MergeTree-engine tables in a background.", 0) \ + M(UInt64, background_common_pool_size, 8, "The maximum number of threads that will be used for performing a variety of operations (mostly garbage collection) for *MergeTree-engine tables in a background.", 0) \ + M(UInt64, background_buffer_flush_schedule_pool_size, 16, "The maximum number of threads that will be used for performing flush operations for Buffer-engine tables in a background.", 0) \ + M(UInt64, background_schedule_pool_size, 16, "The maximum number of threads that will be used for constantly executing some lightweight periodic operations.", 0) \ + M(UInt64, background_message_broker_schedule_pool_size, 16, "The maximum number of threads that will be used for executing background operations for message streaming.", 0) \ + M(UInt64, background_distributed_schedule_pool_size, 16, "The maximum number of threads that will be used for executing distributed sends.", 0) \ + + +DECLARE_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS) + +struct ServerSettings : public BaseSettings +{ + void loadSettingsFromConfig(const Poco::Util::AbstractConfiguration & config); +}; + +} + diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c1b396778ab..3908254b6f1 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -856,6 +856,7 @@ class IColumn; M(UInt64, output_format_parquet_row_group_size, 1000000, "Row group size in rows.", 0) \ M(Bool, output_format_parquet_string_as_string, false, "Use Parquet String type instead of Binary for String columns.", 0) \ M(Bool, output_format_parquet_fixed_string_as_fixed_byte_array, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary for FixedString columns.", 0) \ + M(ParquetVersion, output_format_parquet_version, "2.latest", "Parquet format version for output format. Supported versions: 1.0, 2.4, 2.6 and 2.latest (default)", 0) \ M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \ M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ M(String, output_format_avro_string_column_pattern, "", "For Avro format: regexp of String columns to select as AVRO string.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index c9d68b94a5e..04f328bb665 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -80,7 +80,8 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { - {"23.3", {{"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}}}, + {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, + {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}}}, {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index bce4f7c0000..9e1ab585bb0 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -171,4 +171,12 @@ IMPLEMENT_SETTING_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS, {{"mmap", LocalFSReadMethod::mmap}, {"pread", LocalFSReadMethod::pread}, {"read", LocalFSReadMethod::read}}) + + +IMPLEMENT_SETTING_ENUM_WITH_RENAME(ParquetVersion, ErrorCodes::BAD_ARGUMENTS, + {{"1.0", FormatSettings::ParquetVersion::V1_0}, + {"2.4", FormatSettings::ParquetVersion::V2_4}, + {"2.6", FormatSettings::ParquetVersion::V2_6}, + {"2.latest", FormatSettings::ParquetVersion::V2_LATEST}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 35eb8eb4b6c..139a04f3a5a 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -72,6 +72,8 @@ DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, FormatSettings::DateTimeIn DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeOutputFormat, FormatSettings::DateTimeOutputFormat) +DECLARE_SETTING_ENUM_WITH_RENAME(ParquetVersion, FormatSettings::ParquetVersion) + enum class LogsLevel { none = 0, /// Disable diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 4164bf1e27e..06cd53013ec 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -150,10 +150,16 @@ template struct SettingFieldNumber; template struct SettingFieldNumber; template struct SettingFieldNumber; template struct SettingFieldNumber; +template struct SettingFieldNumber; +template struct SettingFieldNumber; +template struct SettingFieldNumber; template struct SettingAutoWrapper>; template struct SettingAutoWrapper>; template struct SettingAutoWrapper>; +template struct SettingAutoWrapper>; +template struct SettingAutoWrapper>; +template struct SettingAutoWrapper>; namespace { diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index c6fe46c9f6b..3994e402c9a 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -55,7 +55,10 @@ struct SettingFieldNumber using SettingFieldUInt64 = SettingFieldNumber; using SettingFieldInt64 = SettingFieldNumber; +using SettingFieldUInt32 = SettingFieldNumber; +using SettingFieldInt32 = SettingFieldNumber; using SettingFieldFloat = SettingFieldNumber; +using SettingFieldDouble = SettingFieldNumber; using SettingFieldBool = SettingFieldNumber; /** Wraps any SettingField to support special value 'auto' that can be checked with `is_auto` flag. @@ -129,6 +132,9 @@ struct SettingAutoWrapper using SettingFieldUInt64Auto = SettingAutoWrapper; using SettingFieldInt64Auto = SettingAutoWrapper; using SettingFieldFloatAuto = SettingAutoWrapper; +using SettingFieldUInt32Auto = SettingAutoWrapper; +using SettingFieldInt32Auto = SettingAutoWrapper; +using SettingFieldDoubleAuto = SettingAutoWrapper; /* Similar to SettingFieldUInt64Auto with small differences to behave like regular UInt64, supported to compatibility. * When setting to 'auto' it becomes equal to the number of processor cores without taking into account SMT. diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index c636f200324..caba2a52a51 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -99,6 +99,17 @@ struct RegExpTreeDictionary::RegexTreeNode return searcher.Match(haystack, 0, size, re2_st::RE2::Anchor::UNANCHORED, nullptr, 0); } + /// check if this node can cover all the attributes from the query. + bool containsAll(const std::unordered_map & matching_attributes) const + { + for (const auto & [key, value] : matching_attributes) + { + if (!attributes.contains(key)) + return false; + } + return true; + } + struct AttributeValue { Field field; @@ -498,6 +509,9 @@ std::unordered_map RegExpTreeDictionary::match( if (node_ptr->match(reinterpret_cast(keys_data.data()) + offset, length)) { match_result.insertNodeID(node_ptr->id); + /// When this node is leaf and contains all the required attributes, it means a match. + if (node_ptr->containsAll(attributes) && node_ptr->children.empty()) + break; } } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index a22926973ed..aca3166a8c4 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -110,6 +110,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.null_as_default = settings.input_format_null_as_default; format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; + format_settings.parquet.output_version = settings.output_format_parquet_version; format_settings.parquet.import_nested = settings.input_format_parquet_import_nested; format_settings.parquet.case_insensitive_column_matching = settings.input_format_parquet_case_insensitive_column_matching; format_settings.parquet.allow_missing_columns = settings.input_format_parquet_allow_missing_columns; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 5e95c629c7c..d1755a35c5f 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -175,6 +175,14 @@ struct FormatSettings String column_for_object_name; } json_object_each_row; + enum class ParquetVersion + { + V1_0, + V2_4, + V2_6, + V2_LATEST, + }; + struct { UInt64 row_group_size = 1000000; @@ -186,6 +194,7 @@ struct FormatSettings bool output_string_as_string = false; bool output_fixed_string_as_fixed_byte_array = true; UInt64 max_block_size = 8192; + ParquetVersion output_version; } parquet; struct Pretty diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 547df77be6a..5df0d1831af 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -647,11 +647,12 @@ public: case ElementType::OBJECT: type = '{'; break; + case ElementType::BOOL: + type = 'b'; + break; case ElementType::NULL_VALUE: type = 0; break; - default: - return false; } ColumnVector & col_vec = assert_cast &>(dest); diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index dc09facb81b..f1f0fef8fd9 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -55,14 +55,19 @@ private: getName(), arguments.size()); } - for (const auto & arg : arguments) + DataTypes arg_types; + for (size_t i = 0, size = arguments.size(); i < size; ++i) { - if (!isInteger(arg)) + if (i < 2 && WhichDataType(arguments[i]).isIPv4()) + arg_types.emplace_back(std::make_shared()); + else if (isInteger(arguments[i])) + arg_types.push_back(arguments[i]); + else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", - arg->getName(), getName()); + arguments[i]->getName(), getName()); } - DataTypePtr common_type = getLeastSupertype(arguments); + DataTypePtr common_type = getLeastSupertype(arg_types); return std::make_shared(common_type); } diff --git a/src/Functions/widthBucket.cpp b/src/Functions/widthBucket.cpp new file mode 100644 index 00000000000..a32fa159c2c --- /dev/null +++ b/src/Functions/widthBucket.cpp @@ -0,0 +1,290 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + +class FunctionWidthBucket : public IFunction +{ + template + void throwIfInvalid( + const size_t argument_index, + const ColumnConst * col_const, + const typename ColumnVector::Container * col_vec, + const size_t expected_size) const + { + if ((nullptr == col_const) ^ (nullptr != col_vec && col_vec->size() == expected_size)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Logical error in function {}: argument {} has unexpected type or size!", + getName(), + argument_index); + } + } + + template + const typename ColumnVector::Container * getDataIfNotNull(const ColumnVector * col_vec) const + { + if (nullptr == col_vec) + { + return nullptr; + } + return &col_vec->getData(); + } + + template + static TDataType + getValue(const ColumnConst * col_const, const typename ColumnVector::Container * col_vec, const size_t index) + { + if (nullptr != col_const) + { + return col_const->getValue(); + } + return col_vec->data()[index]; + } + + static Float64 calculateRelativeBucket(const Float64 operand, const Float64 low, const Float64 high) + { + return (operand - low) / (high - low); + } + + template + std::optional checkArguments(const Float64 operand, const Float64 low, const Float64 high, const TCountType count) const + { + if (count == 0) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Last argument (count) for function {} cannot be 0.", getName()); + } + if (isNaN(operand) || isNaN(low) || isNaN(high)) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "The first three arguments (operand, low, high) cannot be NaN in function {}", getName()); + } + // operand can be infinity, the following conditions will take care of it + if (!isFinite(low) || !isFinite(high)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The second and third arguments (low, high) cannot be Inf function {}", getName()); + } + if (operand < low || low >= high) + { + return 0; + } + else if (operand >= high) + { + return count + 1; + } + return std::nullopt; + } + + template + TResultType NO_SANITIZE_UNDEFINED calculate(const Float64 operand, const Float64 low, const Float64 high, const TCountType count) const + { + if (const auto maybe_early_return = checkArguments(operand, low, high, count); maybe_early_return.has_value()) + { + return *maybe_early_return; + } + + const auto relative_bucket = calculateRelativeBucket(operand, low, high); + + if (isNaN(relative_bucket) || !isFinite(relative_bucket)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, "The calculation resulted in NaN or Inf which is unexpected in function {}.", getName()); + } + return static_cast(count * relative_bucket + 1); + } + + template TCountType> + ColumnPtr executeForResultType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + { + using ResultType = typename NumberTraits::Construct::Type; + auto common_type = std::make_shared>(); + + std::vector casted_columns; + casted_columns.reserve(3); + for (const auto argument_index : collections::range(0, 3)) + { + casted_columns.push_back(castColumn(arguments[argument_index], common_type)); + } + + const auto * operands_vec = getDataIfNotNull(checkAndGetColumn>(casted_columns[0].get())); + const auto * lows_vec = getDataIfNotNull(checkAndGetColumn>(casted_columns[1].get())); + const auto * highs_vec = getDataIfNotNull(checkAndGetColumn>(casted_columns[2].get())); + const auto * counts_vec = getDataIfNotNull(checkAndGetColumn>(arguments[3].column.get())); + + const auto * operands_col_const = checkAndGetColumnConst>(casted_columns[0].get()); + const auto * lows_col_const = checkAndGetColumnConst>(casted_columns[1].get()); + const auto * highs_col_const = checkAndGetColumnConst>(casted_columns[2].get()); + const auto * counts_col_const = checkAndGetColumnConst>(arguments[3].column.get()); + + throwIfInvalid(0, operands_col_const, operands_vec, input_rows_count); + throwIfInvalid(1, lows_col_const, lows_vec, input_rows_count); + throwIfInvalid(2, highs_col_const, highs_vec, input_rows_count); + throwIfInvalid(4, counts_col_const, counts_vec, input_rows_count); + + const auto are_all_const_cols + = nullptr != operands_col_const && nullptr != lows_col_const && nullptr != highs_col_const && nullptr != counts_col_const; + + + if (are_all_const_cols) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Logical error in function {}: unexpected combination of argument types!", getName()); + } + + auto result_column = ColumnVector::create(); + result_column->reserve(1); + auto & result_data = result_column->getData(); + + for (const auto row_index : collections::range(0, input_rows_count)) + { + const auto operand = getValue(operands_col_const, operands_vec, row_index); + const auto low = getValue(lows_col_const, lows_vec, row_index); + const auto high = getValue(highs_col_const, highs_vec, row_index); + const auto count = getValue(counts_col_const, counts_vec, row_index); + result_data.push_back(calculate(operand, low, high, count)); + } + + return result_column; + } + +public: + static inline const char * name = "widthBucket"; + + explicit FunctionWidthBucket() = default; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 4; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (const auto argument_index : collections::range(0, 3)) + { + if (!isNativeNumber(arguments[argument_index])) + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The first three arguments of function {} must be a Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float32 " + "or Float64.", + getName()); + } + } + if (!WhichDataType(arguments[3]).isNativeUInt()) + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The last argument of function {} must be UInt8, UInt16, UInt32 or UInt64, found {}.", + getName(), + arguments[3]->getName()); + } + switch (arguments[3]->getTypeId()) + { + case TypeIndex::UInt8: + return std::make_shared(); + case TypeIndex::UInt16: + return std::make_shared(); + case TypeIndex::UInt32: + [[fallthrough]]; + case TypeIndex::UInt64: + return std::make_shared(); + default: + break; + } + + UNREACHABLE(); + } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + ColumnPtr + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + switch (arguments[3].type->getTypeId()) + { + case TypeIndex::UInt8: + return executeForResultType(arguments, input_rows_count); + case TypeIndex::UInt16: + return executeForResultType(arguments, input_rows_count); + case TypeIndex::UInt32: + return executeForResultType(arguments, input_rows_count); + case TypeIndex::UInt64: + return executeForResultType(arguments, input_rows_count); + default: + break; + } + + UNREACHABLE(); + } + + bool useDefaultImplementationForConstants() const override { return true; } +}; + +REGISTER_FUNCTION(WidthBucket) +{ + factory.registerFunction({ + R"( +Returns the number of the bucket in which `operand` falls in a histogram having `count` equal-width buckets spanning the range `low` to `high`. Returns `0` if `operand < low`, and returns `count+1` if `operand >= high`. + +`operand`, `low`, `high` can be any native number type. `count` can only be unsigned native integer and its value cannot be zero. + +**Syntax** + +```sql +widthBucket(operand, low, high, count) +``` + +There is also a case insensitive alias called `WIDTH_BUCKET` to provide compatibility with other databases. + +**Example** + +Query: +[example:simple] + +Result: + +``` text +┌─widthBucket(10.15, -8.6, 23, 18)─┐ +│ 11 │ +└──────────────────────────────────┘ +``` +)", + Documentation::Examples{ + {"simple", "SELECT widthBucket(10.15, -8.6, 23, 18)"}, + }, + Documentation::Categories{"Mathematical"}, + }); + + factory.registerAlias("width_bucket", "widthBucket", FunctionFactory::CaseInsensitive); +} + +} diff --git a/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp b/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp index 66e0813b977..ef3e6739f8a 100644 --- a/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp +++ b/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp @@ -73,24 +73,6 @@ Field zeroField(const Field & value) throw Exception(ErrorCodes::BAD_TYPE_OF_FIELD, "Unexpected literal type in function"); } -const String & changeNameIfNeeded(const String & func_name, const String & child_name, const ASTLiteral & literal) -{ - static const std::unordered_map> matches = { - { "min", { "multiply", "divide" } }, - { "max", { "multiply", "divide" } } - }; - - static const std::unordered_map swap_to = { - { "min", "max" }, - { "max", "min" } - }; - - if (literal.value < zeroField(literal.value) && matches.contains(func_name) && matches.find(func_name)->second.contains(child_name)) - return swap_to.find(func_name)->second; - - return func_name; -} - ASTPtr tryExchangeFunctions(const ASTFunction & func) { static const std::unordered_map> supported @@ -114,19 +96,42 @@ ASTPtr tryExchangeFunctions(const ASTFunction & func) ASTPtr optimized_ast; + /** Need reverse max <-> min for: + * + * max(-1*value) -> -1*min(value) + * max(value/-2) -> min(value)/-2 + * max(1-value) -> 1-min(value) + */ + auto get_reverse_aggregate_function_name = [](const std::string & aggregate_function_name) -> std::string + { + if (aggregate_function_name == "min") + return "max"; + else if (aggregate_function_name == "max") + return "min"; + else + return aggregate_function_name; + }; + if (first_literal && !second_literal) { /// It's possible to rewrite 'sum(1/n)' with 'sum(1) * div(1/n)' but we lose accuracy. Ignored. if (child_func->name == "divide") return {}; + bool need_reverse + = (child_func->name == "multiply" && first_literal->value < zeroField(first_literal->value)) || child_func->name == "minus"; + if (need_reverse) + lower_name = get_reverse_aggregate_function_name(lower_name); - const String & new_name = changeNameIfNeeded(lower_name, child_func->name, *first_literal); - optimized_ast = exchangeExtractFirstArgument(new_name, *child_func); + optimized_ast = exchangeExtractFirstArgument(lower_name, *child_func); } else if (second_literal) /// second or both are consts { - const String & new_name = changeNameIfNeeded(lower_name, child_func->name, *second_literal); - optimized_ast = exchangeExtractSecondArgument(new_name, *child_func); + bool need_reverse + = (child_func->name == "multiply" || child_func->name == "divide") && second_literal->value < zeroField(second_literal->value); + if (need_reverse) + lower_name = get_reverse_aggregate_function_name(lower_name); + + optimized_ast = exchangeExtractSecondArgument(lower_name, *child_func); } if (optimized_ast) diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 9e6326b431a..e7302b6324a 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -92,12 +92,11 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con return columns; } -bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescription & columns, const ContextPtr & context) +bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescription & columns, const ContextPtr & context, String & error) { ParserColumnDeclarationList parser(true, true); const Settings & settings = context->getSettingsRef(); - String error; const char * start = structure.data(); const char * end = structure.data() + structure.size(); ASTPtr columns_list_raw = tryParseQuery(parser, start, end, error, false, "columns declaration list", false, settings.max_query_size, settings.max_parser_depth); @@ -106,7 +105,10 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip auto * columns_list = dynamic_cast(columns_list_raw.get()); if (!columns_list) + { + error = fmt::format("Invalid columns declaration list: \"{}\"", structure); return false; + } try { @@ -118,6 +120,7 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip } catch (...) { + error = getCurrentExceptionMessage(false); return false; } } diff --git a/src/Interpreters/parseColumnsListForTableFunction.h b/src/Interpreters/parseColumnsListForTableFunction.h index 97923bcad77..212c378d3b5 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.h +++ b/src/Interpreters/parseColumnsListForTableFunction.h @@ -33,6 +33,6 @@ void validateDataType(const DataTypePtr & type, const DataTypeValidationSettings /// Parses a common argument for table functions such as table structure given in string ColumnsDescription parseColumnsListFromString(const std::string & structure, const ContextPtr & context); -bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescription & columns, const ContextPtr & context); +bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescription & columns, const ContextPtr & context, String & error); } diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index 940030577d6..ba398b995be 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes ASTPtr ASTColumnsRegexpMatcher::clone() const { auto clone = std::make_shared(*this); + clone->children.clear(); if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); } if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); } @@ -91,6 +92,7 @@ bool ASTColumnsRegexpMatcher::isColumnMatching(const String & column_name) const ASTPtr ASTColumnsListMatcher::clone() const { auto clone = std::make_shared(*this); + clone->children.clear(); if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); } if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); } @@ -150,6 +152,7 @@ void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatSt ASTPtr ASTQualifiedColumnsRegexpMatcher::clone() const { auto clone = std::make_shared(*this); + clone->children.clear(); if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); } @@ -216,6 +219,7 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting ASTPtr ASTQualifiedColumnsListMatcher::clone() const { auto clone = std::make_shared(*this); + clone->children.clear(); if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); } diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 59b09f91888..f2259ca7e33 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -334,6 +334,16 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl } else if (query_node || union_node) { + if (table_expression_data.getColumnNames().empty()) + { + const auto & projection_columns = query_node ? query_node->getProjectionColumns() : union_node->computeProjectionColumns(); + NamesAndTypesList projection_columns_list(projection_columns.begin(), projection_columns.end()); + auto additional_column_to_read = ExpressionActions::getSmallestColumn(projection_columns_list); + + const auto & column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(additional_column_to_read, table_expression); + table_expression_data.addColumn(additional_column_to_read, column_identifier); + } + auto subquery_options = select_query_options.subquery(); Planner subquery_planner(table_expression, subquery_options, planner_context->getGlobalPlannerContext()); /// Propagate storage limits to subquery diff --git a/src/Processors/Formats/ISchemaReader.cpp b/src/Processors/Formats/ISchemaReader.cpp index 48cb093f0ab..c96cb373a2d 100644 --- a/src/Processors/Formats/ISchemaReader.cpp +++ b/src/Processors/Formats/ISchemaReader.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -15,20 +16,38 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -void checkFinalInferredType(DataTypePtr & type, const String & name, const FormatSettings & settings, const DataTypePtr & default_type, size_t rows_read) +void checkFinalInferredType( + DataTypePtr & type, + const String & name, + const FormatSettings & settings, + const DataTypePtr & default_type, + size_t rows_read, + const String & hints_parsing_error) { if (!checkIfTypeIsComplete(type)) { if (!default_type) - throw Exception( - ErrorCodes::ONLY_NULLS_WHILE_READING_SCHEMA, - "Cannot determine type for column '{}' by first {} rows " - "of data, most likely this column contains only Nulls or empty " - "Arrays/Maps. You can specify the type for this column using setting schema_inference_hints. " - "If your data contains complex JSON objects, try enabling one " - "of the settings allow_experimental_object_type/input_format_json_read_objects_as_strings", - name, - rows_read); + { + if (hints_parsing_error.empty()) + throw Exception( + ErrorCodes::ONLY_NULLS_WHILE_READING_SCHEMA, + "Cannot determine type for column '{}' by first {} rows " + "of data, most likely this column contains only Nulls or empty " + "Arrays/Maps. You can specify the type for this column using setting schema_inference_hints. " + "If your data contains complex JSON objects, try enabling one " + "of the settings allow_experimental_object_type/input_format_json_read_objects_as_strings", + name, + rows_read); + else + throw Exception( + ErrorCodes::ONLY_NULLS_WHILE_READING_SCHEMA, + "Cannot determine type for column '{}' by first {} rows " + "of data, most likely this column contains only Nulls or empty Arrays/Maps. " + "Column types from setting schema_inference_hints couldn't be parsed because of error: {}", + name, + rows_read, + hints_parsing_error); + } type = default_type; } @@ -46,11 +65,15 @@ IIRowSchemaReader::IIRowSchemaReader(ReadBuffer & in_, const FormatSettings & fo void IIRowSchemaReader::setContext(ContextPtr & context) { ColumnsDescription columns; - if (tryParseColumnsListFromString(hints_str, columns, context)) + if (tryParseColumnsListFromString(hints_str, columns, context, hints_parsing_error)) { for (const auto & [name, type] : columns.getAll()) hints[name] = type; } + else + { + LOG_WARNING(&Poco::Logger::get("IIRowSchemaReader"), "Couldn't parse schema inference hints: {}. This setting will be ignored", hints_parsing_error); + } } void IIRowSchemaReader::transformTypesIfNeeded(DataTypePtr & type, DataTypePtr & new_type) @@ -137,7 +160,14 @@ NamesAndTypesList IRowSchemaReader::readSchema() if (!new_data_types[field_index] || hints.contains(column_names[field_index])) continue; - chooseResultColumnType(*this, data_types[field_index], new_data_types[field_index], getDefaultType(field_index), std::to_string(field_index + 1), rows_read); + chooseResultColumnType( + *this, + data_types[field_index], + new_data_types[field_index], + getDefaultType(field_index), + std::to_string(field_index + 1), + rows_read, + hints_parsing_error); } } @@ -149,7 +179,7 @@ NamesAndTypesList IRowSchemaReader::readSchema() { transformFinalTypeIfNeeded(data_types[field_index]); /// Check that we could determine the type of this column. - checkFinalInferredType(data_types[field_index], column_names[field_index], format_settings, getDefaultType(field_index), rows_read); + checkFinalInferredType(data_types[field_index], column_names[field_index], format_settings, getDefaultType(field_index), rows_read, hints_parsing_error); } result.emplace_back(column_names[field_index], data_types[field_index]); } @@ -246,7 +276,7 @@ NamesAndTypesList IRowWithNamesSchemaReader::readSchema() continue; auto & type = it->second; - chooseResultColumnType(*this, type, new_type, default_type, name, rows_read); + chooseResultColumnType(*this, type, new_type, default_type, name, rows_read, hints_parsing_error); } } @@ -263,7 +293,7 @@ NamesAndTypesList IRowWithNamesSchemaReader::readSchema() { transformFinalTypeIfNeeded(type); /// Check that we could determine the type of this column. - checkFinalInferredType(type, name, format_settings, default_type, rows_read); + checkFinalInferredType(type, name, format_settings, default_type, rows_read, hints_parsing_error); } result.emplace_back(name, type); } diff --git a/src/Processors/Formats/ISchemaReader.h b/src/Processors/Formats/ISchemaReader.h index edc5c6068c3..81bc94afa6c 100644 --- a/src/Processors/Formats/ISchemaReader.h +++ b/src/Processors/Formats/ISchemaReader.h @@ -65,6 +65,7 @@ protected: String hints_str; FormatSettings format_settings; std::unordered_map hints; + String hints_parsing_error; }; /// Base class for schema inference for formats that read data row by row. @@ -145,7 +146,8 @@ void chooseResultColumnType( DataTypePtr & new_type, const DataTypePtr & default_type, const String & column_name, - size_t row) + size_t row, + const String & hints_parsing_error = "") { if (!type) { @@ -166,14 +168,25 @@ void chooseResultColumnType( type = default_type; else { - throw Exception( - ErrorCodes::TYPE_MISMATCH, - "Automatically defined type {} for column '{}' in row {} differs from type defined by previous rows: {}. " - "You can specify the type for this column using setting schema_inference_hints", - type->getName(), - column_name, - row, - new_type->getName()); + if (hints_parsing_error.empty()) + throw Exception( + ErrorCodes::TYPE_MISMATCH, + "Automatically defined type {} for column '{}' in row {} differs from type defined by previous rows: {}. " + "You can specify the type for this column using setting schema_inference_hints", + type->getName(), + column_name, + row, + new_type->getName()); + else + throw Exception( + ErrorCodes::TYPE_MISMATCH, + "Automatically defined type {} for column '{}' in row {} differs from type defined by previous rows: {}. " + "Column types from setting schema_inference_hints couldn't be parsed because of error: {}", + type->getName(), + column_name, + row, + new_type->getName(), + hints_parsing_error); } } @@ -196,7 +209,13 @@ void chooseResultColumnTypes( chooseResultColumnType(schema_reader, types[i], new_types[i], default_type, column_names[i], row); } -void checkFinalInferredType(DataTypePtr & type, const String & name, const FormatSettings & settings, const DataTypePtr & default_type, size_t rows_read); +void checkFinalInferredType( + DataTypePtr & type, + const String & name, + const FormatSettings & settings, + const DataTypePtr & default_type, + size_t rows_read, + const String & hints_parsing_error); Strings splitColumnNames(const String & column_names_str); diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp index 204a5077e31..a39722950e4 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp @@ -182,7 +182,7 @@ JSONColumnsSchemaReaderBase::JSONColumnsSchemaReaderBase( void JSONColumnsSchemaReaderBase::setContext(ContextPtr & ctx) { ColumnsDescription columns; - if (tryParseColumnsListFromString(hints_str, columns, ctx)) + if (tryParseColumnsListFromString(hints_str, columns, ctx, hints_parsing_error)) { for (const auto & [name, type] : columns.getAll()) hints[name] = type; @@ -238,7 +238,7 @@ NamesAndTypesList JSONColumnsSchemaReaderBase::readSchema() rows_in_block = 0; auto column_type = readColumnAndGetDataType( column_name, rows_in_block, format_settings.max_rows_to_read_for_schema_inference - total_rows_read); - chooseResultColumnType(*this, names_to_types[column_name], column_type, nullptr, column_name, total_rows_read + 1); + chooseResultColumnType(*this, names_to_types[column_name], column_type, nullptr, column_name, total_rows_read + 1, hints_parsing_error); } ++iteration; @@ -260,7 +260,7 @@ NamesAndTypesList JSONColumnsSchemaReaderBase::readSchema() { transformJSONTupleToArrayIfPossible(type, format_settings, &inference_info); /// Check that we could determine the type of this column. - checkFinalInferredType(type, name, format_settings, nullptr, format_settings.max_rows_to_read_for_schema_inference); + checkFinalInferredType(type, name, format_settings, nullptr, format_settings.max_rows_to_read_for_schema_inference, hints_parsing_error); } result.emplace_back(name, type); } diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h index 3292b5649c9..2babc0734f9 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h @@ -91,6 +91,7 @@ private: const FormatSettings format_settings; String hints_str; std::unordered_map hints; + String hints_parsing_error; std::unique_ptr reader; Names column_names_from_settings; JSONInferenceInfo inference_info; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 0fce98f8a11..18c81f8fd6a 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -16,6 +16,21 @@ namespace ErrorCodes extern const int UNKNOWN_EXCEPTION; } +static parquet::ParquetVersion::type getParquetVersion(const FormatSettings & settings) +{ + switch (settings.parquet.output_version) + { + case FormatSettings::ParquetVersion::V1_0: + return parquet::ParquetVersion::PARQUET_1_0; + case FormatSettings::ParquetVersion::V2_4: + return parquet::ParquetVersion::PARQUET_2_4; + case FormatSettings::ParquetVersion::V2_6: + return parquet::ParquetVersion::PARQUET_2_6; + case FormatSettings::ParquetVersion::V2_LATEST: + return parquet::ParquetVersion::PARQUET_2_LATEST; + } +} + ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) : IOutputFormat(header_, out_), format_settings{format_settings_} { @@ -44,6 +59,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) auto sink = std::make_shared(out); parquet::WriterProperties::Builder builder; + builder.version(getParquetVersion(format_settings)); #if USE_SNAPPY builder.compression(parquet::Compression::SNAPPY); #endif diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index a58d70a8428..870106d794f 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -387,7 +387,8 @@ Chain buildPushingToViewsChain( chains.emplace_back(std::move(out)); /// Add the view to the query access info so it can appear in system.query_log - if (!no_destination) + /// hasQueryContext - for materialized tables with background replication process query context is not added + if (!no_destination && context->hasQueryContext()) { context->getQueryContext()->addQueryAccessInfo( backQuoteIfNeed(view_id.getDatabaseName()), views_data->views.back().runtime_stats->target_name, {}, "", view_id.getFullTableName()); @@ -757,7 +758,6 @@ IProcessor::Status FinalizingViewsTransform::prepare() output.finish(); return Status::Finished; } - return Status::NeedData; } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index d898049209e..702743ef1f0 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -45,6 +45,9 @@ #include #include +#include + +namespace fs = std::filesystem; #if USE_SSL #include @@ -619,7 +622,7 @@ void HTTPHandler::processQuery( if (buffer_until_eof) { const std::string tmp_path(server.context()->getTemporaryVolume()->getDisk()->getPath()); - const std::string tmp_path_template(tmp_path + "http_buffers/"); + const std::string tmp_path_template(fs::path(tmp_path) / "http_buffers/"); auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) { diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index b671106f46a..c6efe9c9589 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -173,6 +173,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write writeUUIDText(part->uuid, out); String remote_fs_metadata = parse(params.get("remote_fs_metadata", "")); + std::regex re("\\s*,\\s*"); Strings capability( std::sregex_token_iterator(remote_fs_metadata.begin(), remote_fs_metadata.end(), re, -1), @@ -477,6 +478,22 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( int server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); + String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); + + DiskPtr preffered_disk = disk; + + if (!preffered_disk) + { + for (const auto & disk_candidate : data.getDisks()) + { + if (toString(disk_candidate->getDataSourceDescription().type) == remote_fs_metadata) + { + preffered_disk = disk_candidate; + break; + } + } + } + ReservationPtr reservation; size_t sum_files_size = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) @@ -493,31 +510,32 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( if (!disk) { - LOG_TRACE(log, "Disk for fetch is not provided, reserving space using storage balanced reservation"); + LOG_TEST(log, "Disk for fetch is not provided, reserving space using storage balanced reservation"); reservation = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, &ttl_infos, true); + if (!reservation) { - LOG_TRACE(log, "Disk for fetch is not provided, reserving space using TTL rules"); + LOG_TEST(log, "Disk for fetch is not provided, reserving space using TTL rules"); reservation - = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true); + = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true, preffered_disk); } } } else if (!disk) { - LOG_TRACE(log, "Making balanced reservation"); + LOG_TEST(log, "Making balanced reservation"); reservation = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, nullptr); if (!reservation) { - LOG_TRACE(log, "Making simple reservation"); + LOG_TEST(log, "Making simple reservation"); reservation = data.reserveSpace(sum_files_size); } } } else if (!disk) { - LOG_TRACE(log, "Making reservation on the largest disk"); + LOG_TEST(log, "Making reservation on the largest disk"); /// We don't know real size of part because sender server version is too old reservation = data.makeEmptyReservationOnLargestDisk(); } @@ -525,11 +543,11 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( if (!disk) { disk = reservation->getDisk(); - LOG_INFO(log, "Disk for fetch is not provided, getting disk from reservation {} with type {}", disk->getName(), toString(disk->getDataSourceDescription().type)); + LOG_TRACE(log, "Disk for fetch is not provided, getting disk from reservation {} with type '{}'", disk->getName(), toString(disk->getDataSourceDescription().type)); } else { - LOG_INFO(log, "Disk for fetch is disk {} with type {}", disk->getName(), toString(disk->getDataSourceDescription().type)); + LOG_TEST(log, "Disk for fetch is disk {} with type {}", disk->getName(), toString(disk->getDataSourceDescription().type)); } UInt64 revision = parse(in->getResponseCookie("disk_revision", "0")); @@ -552,8 +570,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) readUUIDText(part_uuid, *in); - String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); - size_t projections = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION) readBinary(projections, *in); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e36ab125f97..63374a089ec 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4603,8 +4603,18 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & throw Exception(ErrorCodes::UNKNOWN_DISK, "All parts of partition '{}' are already on disk '{}'", partition_id, disk->getName()); } - if (!movePartsToSpace(parts, std::static_pointer_cast(disk))) - throw Exception(ErrorCodes::ABORTED, "Cannot move parts because moves are manually disabled"); + MovePartsOutcome moves_outcome = movePartsToSpace(parts, std::static_pointer_cast(disk)); + switch (moves_outcome) + { + case MovePartsOutcome::MovesAreCancelled: + throw Exception(ErrorCodes::ABORTED, "Cannot move parts because moves are manually disabled"); + case MovePartsOutcome::NothingToMove: + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No parts to move are found in partition {}", partition_id); + case MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy: + throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Move was not finished, because zero copy mode is enabled and someone other is moving the same parts right now"); + case MovePartsOutcome::PartsMoved: + break; + } } @@ -4656,8 +4666,18 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String throw Exception(ErrorCodes::UNKNOWN_DISK, "All parts of partition '{}' are already on volume '{}'", partition_id, volume->getName()); } - if (!movePartsToSpace(parts, std::static_pointer_cast(volume))) - throw Exception(ErrorCodes::ABORTED, "Cannot move parts because moves are manually disabled"); + MovePartsOutcome moves_outcome = movePartsToSpace(parts, std::static_pointer_cast(volume)); + switch (moves_outcome) + { + case MovePartsOutcome::MovesAreCancelled: + throw Exception(ErrorCodes::ABORTED, "Cannot move parts because moves are manually disabled"); + case MovePartsOutcome::NothingToMove: + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No parts to move are found in partition {}", partition_id); + case MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy: + throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Move was not finished, because zero copy mode is enabled and someone other is moving the same parts right now"); + case MovePartsOutcome::PartsMoved: + break; + } } void MergeTreeData::movePartitionToShard(const ASTPtr & /*partition*/, bool /*move_part*/, const String & /*to*/, ContextPtr /*query_context*/) @@ -7442,7 +7462,7 @@ bool MergeTreeData::scheduleDataMovingJob(BackgroundJobsAssignee & assignee) assignee.scheduleMoveTask(std::make_shared( [this, moving_tagger] () mutable { - return moveParts(moving_tagger); + return moveParts(moving_tagger) == MovePartsOutcome::PartsMoved; }, moves_assignee_trigger, getStorageID())); return true; } @@ -7457,14 +7477,14 @@ bool MergeTreeData::areBackgroundMovesNeeded() const return policy->getVolumes().size() == 1 && policy->getVolumes()[0]->getDisks().size() > 1; } -bool MergeTreeData::movePartsToSpace(const DataPartsVector & parts, SpacePtr space) +MovePartsOutcome MergeTreeData::movePartsToSpace(const DataPartsVector & parts, SpacePtr space) { if (parts_mover.moves_blocker.isCancelled()) - return false; + return MovePartsOutcome::MovesAreCancelled; auto moving_tagger = checkPartsForMove(parts, space); if (moving_tagger->parts_to_move.empty()) - return false; + return MovePartsOutcome::NothingToMove; return moveParts(moving_tagger); } @@ -7521,13 +7541,13 @@ MergeTreeData::CurrentlyMovingPartsTaggerPtr MergeTreeData::checkPartsForMove(co return std::make_shared(std::move(parts_to_move), *this); } -bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger) +MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger) { LOG_INFO(log, "Got {} parts to move.", moving_tagger->parts_to_move.size()); const auto settings = getSettings(); - bool result = true; + MovePartsOutcome result{MovePartsOutcome::PartsMoved}; for (const auto & moving_part : moving_tagger->parts_to_move) { Stopwatch stopwatch; @@ -7583,7 +7603,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge { /// Move will be retried but with backoff. LOG_DEBUG(log, "Move of part {} postponed, because zero copy mode enabled and someone other moving this part right now", moving_part.part->name); - result = false; + result = MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy; continue; } } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index faee69b137f..f7fa4f192cc 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1315,7 +1315,7 @@ protected: /// MergeTree because they store mutations in different way. virtual MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; /// Moves part to specified space, used in ALTER ... MOVE ... queries - bool movePartsToSpace(const DataPartsVector & parts, SpacePtr space); + MovePartsOutcome movePartsToSpace(const DataPartsVector & parts, SpacePtr space); /// Makes backup entries to backup the parts of this table. BackupEntries backupParts(const DataPartsVector & data_parts, const String & data_path_in_backup, const ContextPtr & local_context); @@ -1456,7 +1456,7 @@ private: using CurrentlyMovingPartsTaggerPtr = std::shared_ptr; /// Move selected parts to corresponding disks - bool moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger); + MovePartsOutcome moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger); /// Select parts for move and disks for them. Used in background moving processes. CurrentlyMovingPartsTaggerPtr selectPartsForMove(); diff --git a/src/Storages/MergeTree/MergeTreePartsMover.h b/src/Storages/MergeTree/MergeTreePartsMover.h index dfb4bb954d7..1cee98bcba9 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.h +++ b/src/Storages/MergeTree/MergeTreePartsMover.h @@ -11,6 +11,13 @@ namespace DB { +enum class MovePartsOutcome +{ + PartsMoved, + NothingToMove, + MovesAreCancelled, + MoveWasPostponedBecauseOfZeroCopy, +}; /// Active part from storage and destination reservation where it has to be moved struct MergeTreeMoveEntry diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 933ad8e913a..6d280f3aaec 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -43,7 +43,7 @@ struct Settings; M(UInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \ M(UInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "Maximum in total size of parts to merge, when there are maximum free threads in background pool (or entries in replication queue).", 0) \ M(UInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024, "Maximum in total size of parts to merge, when there are minimum free threads in background pool (or entries in replication queue).", 0) \ - M(UInt64, max_replicated_merges_in_queue, 16, "How many tasks of merging and mutating parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \ + M(UInt64, max_replicated_merges_in_queue, 1000, "How many tasks of merging and mutating parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \ M(UInt64, max_replicated_mutations_in_queue, 8, "How many tasks of mutating parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \ M(UInt64, max_replicated_merges_with_ttl_in_queue, 1, "How many tasks of merging parts with TTL are allowed simultaneously in ReplicatedMergeTree queue.", 0) \ M(UInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue). This is to allow small merges to process - not filling the pool with long running merges.", 0) \ diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 494f9c9c31f..66e610ca653 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -8,10 +8,251 @@ #include #include #include +#include +#include + +#include namespace DB { +namespace +{ + +void calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & from, UInt64 & total_size) +{ + /// Files or directories of detached part may not exist. Only count the size of existing files. + if (disk->isFile(from)) + { + total_size += disk->getFileSize(from); + } + else + { + for (auto it = disk->iterateDirectory(from); it->isValid(); it->next()) + calculateTotalSizeOnDiskImpl(disk, fs::path(from) / it->name(), total_size); + } +} + +UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk, const String & from) +{ + UInt64 total_size = 0; + try + { + calculateTotalSizeOnDiskImpl(disk, from, total_size); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + return total_size; +} + +class SourceState +{ + std::mutex mutex; + StoragesInfoStream stream; + +public: + explicit SourceState(StoragesInfoStream && stream_) + : stream(std::move(stream_)) + {} + + StoragesInfo next() + { + std::lock_guard lock(mutex); + return stream.next(); + } +}; + + +struct WorkerState +{ + struct Task + { + DiskPtr disk; + String path; + std::atomic * counter = nullptr; + }; + + std::vector tasks; + std::atomic next_task = {0}; +}; + +class DetachedPartsSource : public ISource +{ +public: + DetachedPartsSource(Block header_, std::shared_ptr state_, std::vector columns_mask_, UInt64 block_size_, + bool has_bytes_on_disk_column_) + : ISource(std::move(header_)) + , state(state_) + , columns_mask(std::move(columns_mask_)) + , block_size(block_size_) + , has_bytes_on_disk_column(has_bytes_on_disk_column_) + {} + + String getName() const override { return "DataPartsSource"; } + +protected: + static Chunk nullWhenNoRows(MutableColumns && new_columns) + { + chassert(!new_columns.empty()); + const auto rows = new_columns[0]->size(); + + if (!rows) + return {}; + + return {std::move(new_columns), rows}; + } + + Chunk generate() override + { + MutableColumns new_columns = getPort().getHeader().cloneEmptyColumns(); + chassert(!new_columns.empty()); + + while (new_columns[0]->size() < block_size) + { + if (detached_parts.empty()) + getMoreParts(); + + if (detached_parts.empty()) + return nullWhenNoRows(std::move(new_columns)); + + generateRows(new_columns, block_size - new_columns[0]->size()); + } + + return nullWhenNoRows(std::move(new_columns)); + } + +private: + std::shared_ptr state; + const std::vector columns_mask; + const UInt64 block_size; + const bool has_bytes_on_disk_column; + const size_t support_threads = 35; + + StoragesInfo current_info; + DetachedPartsInfo detached_parts; + + void getMoreParts() + { + chassert(detached_parts.empty()); + + while (detached_parts.empty()) + { + current_info = state->next(); + if (!current_info) + return; + + detached_parts = current_info.data->getDetachedParts(); + } + } + + void calculatePartSizeOnDisk(size_t begin, std::vector> & parts_sizes) + { + if (!has_bytes_on_disk_column) + return; + + WorkerState worker_state; + + for (auto p_id = begin; p_id < detached_parts.size(); ++p_id) + { + auto & part = detached_parts[p_id]; + auto part_path = fs::path(MergeTreeData::DETACHED_DIR_NAME) / part.dir_name; + auto relative_path = fs::path(current_info.data->getRelativeDataPath()) / part_path; + worker_state.tasks.push_back({part.disk, relative_path, &parts_sizes.at(p_id - begin)}); + } + + std::vector> futures; + SCOPE_EXIT_SAFE({ + /// Cancel all workers + worker_state.next_task.store(worker_state.tasks.size()); + /// Exceptions are not propagated + for (auto & future : futures) + if (future.valid()) + future.wait(); + futures.clear(); + }); + + auto max_thread_to_run = std::max(size_t(1), std::min(support_threads, worker_state.tasks.size() / 10)); + futures.reserve(max_thread_to_run); + + for (size_t i = 0; i < max_thread_to_run; ++i) + { + if (worker_state.next_task.load() >= worker_state.tasks.size()) + break; + + auto worker = [&worker_state] () + { + for (auto id = worker_state.next_task++; id < worker_state.tasks.size(); id = worker_state.next_task++) + { + auto & task = worker_state.tasks.at(id); + size_t size = calculateTotalSizeOnDisk(task.disk, task.path); + task.counter->store(size); + } + }; + + futures.push_back( + scheduleFromThreadPool( + std::move(worker), + IOThreadPool::get(), + "DP_BytesOnDisk")); + } + + /// Exceptions are propagated + for (auto & future : futures) + future.get(); + } + + void generateRows(MutableColumns & new_columns, size_t max_rows) + { + chassert(current_info); + + auto rows = std::min(max_rows, detached_parts.size()); + auto begin = detached_parts.size() - rows; + + std::vector> parts_sizes(rows); + calculatePartSizeOnDisk(begin, parts_sizes); + + for (auto p_id = begin; p_id < detached_parts.size(); ++p_id) + { + auto & p = detached_parts.at(p_id); + + size_t src_index = 0; + size_t res_index = 0; + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(current_info.database); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(current_info.table); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.valid_name ? p.partition_id : Field()); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.dir_name); + if (columns_mask[src_index++]) + { + chassert(has_bytes_on_disk_column); + size_t bytes_on_disk = parts_sizes.at(p_id - begin).load(); + new_columns[res_index++]->insert(bytes_on_disk); + } + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.disk->getName()); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert((fs::path(current_info.data->getFullPathOnDisk(p.disk)) / MergeTreeData::DETACHED_DIR_NAME / p.dir_name).string()); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.valid_name ? p.prefix : Field()); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.valid_name ? p.min_block : Field()); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.valid_name ? p.max_block : Field()); + if (columns_mask[src_index++]) + new_columns[res_index++]->insert(p.valid_name ? p.level : Field()); + } + + detached_parts.resize(begin); + } +}; + +} + StorageSystemDetachedParts::StorageSystemDetachedParts(const StorageID & table_id_) : IStorage(table_id_) { @@ -31,33 +272,6 @@ StorageSystemDetachedParts::StorageSystemDetachedParts(const StorageID & table_i }}); setInMemoryMetadata(storage_metadata); } -static void calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & from, UInt64 & total_size) -{ - /// Files or directories of detached part may not exist. Only count the size of existing files. - if (disk->isFile(from)) - { - total_size += disk->getFileSize(from); - } - else - { - for (auto it = disk->iterateDirectory(from); it->isValid(); it->next()) - calculateTotalSizeOnDiskImpl(disk, fs::path(from) / it->name(), total_size); - } -} - -static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk, const String & from) -{ - UInt64 total_size = 0; - try - { - calculateTotalSizeOnDiskImpl(disk, from, total_size); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - return total_size; -} Pipe StorageSystemDetachedParts::read( const Names & column_names, @@ -65,66 +279,39 @@ Pipe StorageSystemDetachedParts::read( SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, - const size_t /*max_block_size*/, - const size_t /*num_streams*/) + const size_t max_block_size, + const size_t num_streams) { storage_snapshot->check(column_names); - - StoragesInfoStream stream(query_info, context); - - /// Create the result. - Block block = storage_snapshot->metadata->getSampleBlock(); + Block sample_block = storage_snapshot->metadata->getSampleBlock(); NameSet names_set(column_names.begin(), column_names.end()); - std::vector columns_mask(block.columns()); - Block header; - for (size_t i = 0; i < block.columns(); ++i) + Block header; + std::vector columns_mask(sample_block.columns()); + + for (size_t i = 0; i < columns_mask.size(); ++i) { - if (names_set.contains(block.getByPosition(i).name)) + if (names_set.contains(sample_block.getByPosition(i).name)) { columns_mask[i] = 1; - header.insert(block.getByPosition(i)); + header.insert(sample_block.getByPosition(i)); } } - MutableColumns new_columns = header.cloneEmptyColumns(); - while (StoragesInfo info = stream.next()) + bool has_bytes_on_disk_column = names_set.contains("bytes_on_disk"); + + auto state = std::make_shared(StoragesInfoStream(query_info, context)); + + Pipe pipe; + + for (size_t i = 0; i < num_streams; ++i) { - const auto parts = info.data->getDetachedParts(); - for (const auto & p : parts) - { - size_t src_index = 0, res_index = 0; - String detached_part_path = fs::path(MergeTreeData::DETACHED_DIR_NAME) / p.dir_name; - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(info.database); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(info.table); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.valid_name ? p.partition_id : Field()); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.dir_name); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(calculateTotalSizeOnDisk(p.disk, fs::path(info.data->getRelativeDataPath()) / detached_part_path)); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.disk->getName()); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert((fs::path(info.data->getFullPathOnDisk(p.disk)) / detached_part_path).string()); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.valid_name ? p.prefix : Field()); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.valid_name ? p.min_block : Field()); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.valid_name ? p.max_block : Field()); - if (columns_mask[src_index++]) - new_columns[res_index++]->insert(p.valid_name ? p.level : Field()); - } + auto source = std::make_shared(header.cloneEmpty(), state, columns_mask, max_block_size, has_bytes_on_disk_column); + pipe.addSource(std::move(source)); } - UInt64 num_rows = new_columns.at(0)->size(); - Chunk chunk(std::move(new_columns), num_rows); - - return Pipe(std::make_shared(std::move(header), std::move(chunk))); + return pipe; } } diff --git a/src/Storages/System/StorageSystemServerSettings.cpp b/src/Storages/System/StorageSystemServerSettings.cpp new file mode 100644 index 00000000000..ad52c6896ac --- /dev/null +++ b/src/Storages/System/StorageSystemServerSettings.cpp @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +NamesAndTypesList StorageSystemServerSettings::getNamesAndTypes() +{ + return { + {"name", std::make_shared()}, + {"value", std::make_shared()}, + {"default", std::make_shared()}, + {"changed", std::make_shared()}, + {"description", std::make_shared()}, + {"type", std::make_shared()}, + }; +} + +void StorageSystemServerSettings::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const +{ + const auto & config = context->getConfigRef(); + ServerSettings settings; + settings.loadSettingsFromConfig(config); + + for (const auto & setting : settings.all()) + { + const auto & setting_name = setting.getName(); + res_columns[0]->insert(setting_name); + res_columns[1]->insert(setting.getValueString()); + res_columns[2]->insert(setting.getDefaultValueString()); + res_columns[3]->insert(setting.isValueChanged()); + res_columns[4]->insert(setting.getDescription()); + res_columns[5]->insert(setting.getTypeName()); + } +} + +} diff --git a/src/Storages/System/StorageSystemServerSettings.h b/src/Storages/System/StorageSystemServerSettings.h new file mode 100644 index 00000000000..b3aa8055853 --- /dev/null +++ b/src/Storages/System/StorageSystemServerSettings.h @@ -0,0 +1,27 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + + +/** implements system table "settings", which allows to get information about the current settings. + */ +class StorageSystemServerSettings final : public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemServerSettings"; } + + static NamesAndTypesList getNamesAndTypes(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + + void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index bf3a4842187..e13f6c308c4 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -19,6 +19,7 @@ NamesAndTypesList StorageSystemSettings::getNamesAndTypes() {"max", std::make_shared(std::make_shared())}, {"readonly", std::make_shared()}, {"type", std::make_shared()}, + {"default", std::make_shared()}, {"alias_for", std::make_shared()}, }; } @@ -53,6 +54,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[5]->insert(max); res_columns[6]->insert(writability == SettingConstraintWritability::CONST); res_columns[7]->insert(setting.getTypeName()); + res_columns[8]->insert(setting.getDefaultValueString()); }; const auto & settings_to_aliases = Settings::Traits::settingsToAliases(); @@ -62,7 +64,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co res_columns[0]->insert(setting_name); fill_data_for_setting(setting_name, setting); - res_columns[8]->insert(""); + res_columns[9]->insert(""); if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end()) { @@ -70,7 +72,7 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co { res_columns[0]->insert(alias); fill_data_for_setting(alias, setting); - res_columns[8]->insert(setting_name); + res_columns[9]->insert(setting_name); } } } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 07db151069f..61329ab834b 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -105,6 +106,7 @@ void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database) attach(context, system_database, "functions"); attach(context, system_database, "events"); attach(context, system_database, "settings"); + attach(context, system_database, "server_settings"); attach(context, system_database, "settings_changes"); attach>(context, system_database, "merge_tree_settings"); attach>(context, system_database, "replicated_merge_tree_settings"); diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index 545427f30c9..cf0e20c624c 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -92,7 +92,8 @@ void TableFunctionValues::parseArguments(const ASTPtr & ast_function, ContextPtr const auto & literal = args[0]->as(); String value; - if (args.size() > 1 && literal && literal->value.tryGet(value) && tryParseColumnsListFromString(value, structure, context)) + String error; + if (args.size() > 1 && literal && literal->value.tryGet(value) && tryParseColumnsListFromString(value, structure, context, error)) { has_structure_in_arguments = true; return; diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index 7987ea2643f..585cfd52e87 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -52,7 +52,7 @@ class Labels: class ReleaseBranch: - CHERRYPICK_DESCRIPTION = """This pull-request is a first step of an automated \ + CHERRYPICK_DESCRIPTION = f"""This pull-request is a first step of an automated \ backporting. It contains changes like after calling a local command `git cherry-pick`. If you intend to continue backporting this changes, then resolve all conflicts if any. @@ -60,13 +60,16 @@ Otherwise, if you do not want to backport them, then just close this pull-reques The check results does not matter at this step - you can safely ignore them. Also this pull-request will be merged automatically as it reaches the mergeable state, \ - but you always can merge it manually. +**do not merge it manually**. + +If it stuck, check the original PR for `{Labels.BACKPORTS_CREATED}` and delete it if \ +necessary. """ BACKPORT_DESCRIPTION = """This pull-request is a last step of an automated \ backporting. Treat it as a standard pull-request: look at the checks and resolve conflicts. Merge it only if you intend to backport changes to the target branch, otherwise just \ - close it. +close it. """ REMOTE = "" diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 05742acb314..08cd2d466d0 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -289,6 +289,18 @@ CI_CONFIG = { "Stress test (debug)": { "required_build": "package_debug", }, + "Upgrade check (asan)": { + "required_build": "package_asan", + }, + "Upgrade check (tsan)": { + "required_build": "package_tsan", + }, + "Upgrade check (msan)": { + "required_build": "package_msan", + }, + "Upgrade check (debug)": { + "required_build": "package_debug", + }, "Integration tests (asan)": { "required_build": "package_asan", }, diff --git a/docker/test/stress/stress b/tests/ci/stress.py similarity index 90% rename from docker/test/stress/stress rename to tests/ci/stress.py index 4ed8ec838f2..4f723dba101 100755 --- a/docker/test/stress/stress +++ b/tests/ci/stress.py @@ -8,13 +8,13 @@ import logging import time -def get_options(i, backward_compatibility_check): +def get_options(i, upgrade_check): options = [] client_options = [] - if 0 < i: + if i > 0: options.append("--order=random") - if i % 3 == 2 and not backward_compatibility_check: + if i % 3 == 2 and not upgrade_check: options.append( '''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i) ) @@ -30,26 +30,24 @@ def get_options(i, backward_compatibility_check): if i % 2 == 1: join_alg_num = i // 2 - if join_alg_num % 5 == 0: + if join_alg_num % 4 == 0: client_options.append("join_algorithm='parallel_hash'") - if join_alg_num % 5 == 1: + if join_alg_num % 4 == 1: client_options.append("join_algorithm='partial_merge'") - if join_alg_num % 5 == 2: + if join_alg_num % 4 == 2: client_options.append("join_algorithm='full_sorting_merge'") - if join_alg_num % 5 == 3: - client_options.append("join_algorithm='grace_hash'") - if join_alg_num % 5 == 4: + if join_alg_num % 4 == 3: client_options.append("join_algorithm='auto'") - client_options.append('max_rows_in_join=1000') + client_options.append("max_rows_in_join=1000") if i % 5 == 1: client_options.append("memory_tracker_fault_probability=0.001") - if i % 2 == 1 and not backward_compatibility_check: + if i % 2 == 1 and not upgrade_check: client_options.append("group_by_use_nulls=1") # 12 % 3 == 0, so it's Atomic database - if i == 12 and not backward_compatibility_check: + if i == 12 and not upgrade_check: client_options.append("implicit_transaction=1") client_options.append("throw_on_unsupported_query_inside_transaction=0") @@ -65,11 +63,9 @@ def run_func_test( num_processes, skip_tests_option, global_time_limit, - backward_compatibility_check, + upgrade_check, ): - backward_compatibility_check_option = ( - "--backward-compatibility-check" if backward_compatibility_check else "" - ) + upgrade_check_option = "--upgrade-check" if upgrade_check else "" global_time_limit_option = "" if global_time_limit: global_time_limit_option = "--global_time_limit={}".format(global_time_limit) @@ -79,14 +75,14 @@ def run_func_test( for i in range(num_processes) ] pipes = [] - for i in range(0, len(output_paths)): - f = open(output_paths[i], "w") + for i, path in enumerate(output_paths): + f = open(path, "w") full_command = "{} {} {} {} {}".format( cmd, - get_options(i, backward_compatibility_check), + get_options(i, upgrade_check), global_time_limit_option, skip_tests_option, - backward_compatibility_check_option, + upgrade_check_option, ) logging.info("Run func tests '%s'", full_command) p = Popen(full_command, shell=True, stdout=f, stderr=f) @@ -178,7 +174,7 @@ def prepare_for_hung_check(drop_databases): for db in databases: if db == "system": continue - command = make_query_command(f'DETACH DATABASE {db}') + command = make_query_command(f"DETACH DATABASE {db}") # we don't wait for drop Popen(command, shell=True) break @@ -237,7 +233,7 @@ if __name__ == "__main__": parser.add_argument("--output-folder") parser.add_argument("--global-time-limit", type=int, default=1800) parser.add_argument("--num-parallel", type=int, default=cpu_count()) - parser.add_argument("--backward-compatibility-check", action="store_true") + parser.add_argument("--upgrade-check", action="store_true") parser.add_argument("--hung-check", action="store_true", default=False) # make sense only for hung check parser.add_argument("--drop-databases", action="store_true", default=False) @@ -252,7 +248,7 @@ if __name__ == "__main__": args.num_parallel, args.skip_func_tests, args.global_time_limit, - args.backward_compatibility_check, + args.upgrade_check, ) logging.info("Will wait functests to finish") @@ -305,11 +301,12 @@ if __name__ == "__main__": ] ) hung_check_log = os.path.join(args.output_folder, "hung_check.log") - tee = Popen(['/usr/bin/tee', hung_check_log], stdin=PIPE) + tee = Popen(["/usr/bin/tee", hung_check_log], stdin=PIPE) res = call(cmd, shell=True, stdout=tee.stdin, stderr=STDOUT) - tee.stdin.close() + if tee.stdin is not None: + tee.stdin.close() if res != 0 and have_long_running_queries: - logging.info("Hung check failed with exit code {}".format(res)) + logging.info("Hung check failed with exit code %d", res) else: hung_check_status = "No queries hung\tOK\t\\N\t\n" with open( @@ -318,5 +315,4 @@ if __name__ == "__main__": results.write(hung_check_status) os.remove(hung_check_log) - logging.info("Stress test finished") diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index fb38969cb23..7596a81ebc9 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -36,7 +36,6 @@ def get_run_command( "docker run --cap-add=SYS_PTRACE " # a static link, don't use S3_URL or S3_DOWNLOAD "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' " - f"-e DISABLE_BC_CHECK={os.environ.get('DISABLE_BC_CHECK', '0')} " # For dmesg and sysctl "--privileged " f"--volume={build_path}:/package_folder " @@ -109,7 +108,7 @@ def process_results( return state, description, test_results, additional_files -def main(): +def run_stress_test(docker_image_name): logging.basicConfig(level=logging.INFO) stopwatch = Stopwatch() @@ -132,7 +131,7 @@ def main(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(reports_path, "clickhouse/stress-test") + docker_image = get_image_with_version(reports_path, docker_image_name) packages_path = os.path.join(temp_path, "packages") if not os.path.exists(packages_path): @@ -199,4 +198,4 @@ def main(): if __name__ == "__main__": - main() + run_stress_test("clickhouse/stress-test") diff --git a/tests/ci/stress_tests.lib b/tests/ci/stress_tests.lib new file mode 100644 index 00000000000..2cbae112b7f --- /dev/null +++ b/tests/ci/stress_tests.lib @@ -0,0 +1,309 @@ +#!/bin/bash + +# core.COMM.PID-TID +sysctl kernel.core_pattern='core.%e.%p-%P' + +OK="\tOK\t\\N\t" +FAIL="\tFAIL\t\\N\t" + +FAILURE_CONTEXT_LINES=50 +FAILURE_CONTEXT_MAX_LINE_WIDTH=400 + +function escaped() +{ + # That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language. + # Also limit lines width just in case (too long lines are not really useful usually) + clickhouse local -S 's String' --input-format=LineAsString -q "select substr(s, 1, $FAILURE_CONTEXT_MAX_LINE_WIDTH) + from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'" +} + +function head_escaped() +{ + head -n $FAILURE_CONTEXT_LINES $1 | escaped +} + +function unts() +{ + grep -Po "[0-9][0-9]:[0-9][0-9] \K.*" +} + +function trim_server_logs() +{ + head -n $FAILURE_CONTEXT_LINES "/test_output/$1" | grep -Eo " \[ [0-9]+ \] \{.*" | escaped +} + +function install_packages() +{ + dpkg -i $1/clickhouse-common-static_*.deb + dpkg -i $1/clickhouse-common-static-dbg_*.deb + dpkg -i $1/clickhouse-server_*.deb + dpkg -i $1/clickhouse-client_*.deb +} + +function configure() +{ + # install test configs + export USE_DATABASE_ORDINARY=1 + export EXPORT_S3_STORAGE_POLICIES=1 + /usr/share/clickhouse-test/config/install.sh + + # avoid too slow startup + sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ + | sed "s|100000|10000|" \ + > /etc/clickhouse-server/config.d/keeper_port.xml.tmp + sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml + sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml + sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml + + # for clickhouse-server (via service) + echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment + # for clickhouse-client + export ASAN_OPTIONS='malloc_context_size=10 allocator_release_to_os_interval_ms=10000' + + # since we run clickhouse from root + sudo chown root: /var/lib/clickhouse + + # Set more frequent update period of asynchronous metrics to more frequently update information about real memory usage (less chance of OOM). + echo "1" \ + > /etc/clickhouse-server/config.d/asynchronous_metrics_update_period_s.xml + + local total_mem + total_mem=$(awk '/MemTotal/ { print $(NF-1) }' /proc/meminfo) # KiB + total_mem=$(( total_mem*1024 )) # bytes + + # Set maximum memory usage as half of total memory (less chance of OOM). + # + # But not via max_server_memory_usage but via max_memory_usage_for_user, + # so that we can override this setting and execute service queries, like: + # - hung check + # - show/drop database + # - ... + # + # So max_memory_usage_for_user will be a soft limit, and + # max_server_memory_usage will be hard limit, and queries that should be + # executed regardless memory limits will use max_memory_usage_for_user=0, + # instead of relying on max_untracked_memory + + max_server_memory_usage_to_ram_ratio=0.5 + echo "Setting max_server_memory_usage_to_ram_ratio to ${max_server_memory_usage_to_ram_ratio}" + cat > /etc/clickhouse-server/config.d/max_server_memory_usage.xml < + ${max_server_memory_usage_to_ram_ratio} + +EOL + + local max_users_mem + max_users_mem=$((total_mem*30/100)) # 30% + echo "Setting max_memory_usage_for_user=$max_users_mem and max_memory_usage for queries to 10G" + cat > /etc/clickhouse-server/users.d/max_memory_usage_for_user.xml < + + + 10G + ${max_users_mem} + + + +EOL + + cat > /etc/clickhouse-server/config.d/core.xml < + + + 107374182400 + + + $PWD + +EOL + + # Analyzer is not yet ready for testing + cat > /etc/clickhouse-server/users.d/no_analyzer.xml < + + + + + + + + + + +EOL + +} + +function stop() +{ + local max_tries="${1:-90}" + local check_hang="${2:-true}" + local pid + # Preserve the pid, since the server can hung after the PID will be deleted. + pid="$(cat /var/run/clickhouse-server/clickhouse-server.pid)" + + clickhouse stop --max-tries "$max_tries" --do-not-kill && return + + if [ $check_hang == true ] + then + # We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces. + echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv + kill -TERM "$(pidof gdb)" ||: + sleep 5 + echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log + timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$pid" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log + clickhouse stop --force + fi +} + +function start() +{ + counter=0 + until clickhouse-client --query "SELECT 1" + do + if [ "$counter" -gt ${1:-120} ] + then + echo "Cannot start clickhouse-server" + rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt ||: + echo -e "Cannot start clickhouse-server$FAIL$(trim_server_logs application_errors.txt)" >> /test_output/test_results.tsv + cat /var/log/clickhouse-server/stdout.log + tail -n100 /var/log/clickhouse-server/stderr.log + tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e ' RaftInstance:' -e ' RaftInstance' | tail -n100 + break + fi + # use root to match with current uid + clickhouse start --user root >/var/log/clickhouse-server/stdout.log 2>>/var/log/clickhouse-server/stderr.log + sleep 0.5 + counter=$((counter + 1)) + done + + # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog + # and clickhouse-server can do fork-exec, for example, to run some bridge. + # Do not set nostop noprint for all signals, because some it may cause gdb to hang, + # explicitly ignore non-fatal signals that are used by server. + # Number of SIGRTMIN can be determined only in runtime. + RTMIN=$(kill -l SIGRTMIN) + echo " +set follow-fork-mode parent +handle SIGHUP nostop noprint pass +handle SIGINT nostop noprint pass +handle SIGQUIT nostop noprint pass +handle SIGPIPE nostop noprint pass +handle SIGTERM nostop noprint pass +handle SIGUSR1 nostop noprint pass +handle SIGUSR2 nostop noprint pass +handle SIG$RTMIN nostop noprint pass +info signals +continue +backtrace full +thread apply all backtrace full +info registers +disassemble /s +up +disassemble /s +up +disassemble /s +p \"done\" +detach +quit +" > script.gdb + + # FIXME Hung check may work incorrectly because of attached gdb + # 1. False positives are possible + # 2. We cannot attach another gdb to get stacktraces if some queries hung + gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log & + sleep 5 + # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s) + time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||: +} + +function check_server_start() +{ + clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \ + || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \ + && echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(trim_server_logs application_errors.txt)" \ + >> /test_output/test_results.tsv) + + # Remove file application_errors.txt if it's empty + [ -s /test_output/application_errors.txt ] || rm /test_output/application_errors.txt +} + +function check_logs_for_critical_errors() +{ + # Sanitizer asserts + rg -Fa "==================" /var/log/clickhouse-server/stderr.log | rg -v "in query:" >> /test_output/tmp + rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp + rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \ + && echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \ + || echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv + rm -f /test_output/tmp + + # OOM + rg -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \ + && echo -e "Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \ + || echo -e "No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv + + # Logical errors + rg -Fa "Code: 49. DB::Exception: " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/logical_errors.txt \ + && echo -e "Logical error thrown (see clickhouse-server.log or logical_errors.txt)$FAIL$(head_escaped /test_output/logical_errors.txt)" >> /test_output/test_results.tsv \ + || echo -e "No logical errors$OK" >> /test_output/test_results.tsv + # Remove file logical_errors.txt if it's empty + [ -s /test_output/logical_errors.txt ] || rm /test_output/logical_errors.txt + + # No such key errors + rg --text "Code: 499.*The specified key does not exist" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/no_such_key_errors.txt \ + && echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(trim_server_logs no_such_key_errors.txt)" >> /test_output/test_results.tsv \ + || echo -e "No lost s3 keys$OK" >> /test_output/test_results.tsv + + # Remove file no_such_key_errors.txt if it's empty + [ -s /test_output/no_such_key_errors.txt ] || rm /test_output/no_such_key_errors.txt + + # Crash + rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \ + && echo -e "Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \ + || echo -e "Not crashed$OK" >> /test_output/test_results.tsv + + # It also checks for crash without stacktrace (printed by watchdog) + rg -Fa " " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/fatal_messages.txt \ + && echo -e "Fatal message in clickhouse-server.log (see fatal_messages.txt)$FAIL$(trim_server_logs fatal_messages.txt)" >> /test_output/test_results.tsv \ + || echo -e "No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv + + # Remove file fatal_messages.txt if it's empty + [ -s /test_output/fatal_messages.txt ] || rm /test_output/fatal_messages.txt + + rg -Fa "########################################" /test_output/* > /dev/null \ + && echo -e "Killed by signal (output files)$FAIL" >> /test_output/test_results.tsv + + function get_gdb_log_context() + { + rg -A50 -Fa " received signal " /test_output/gdb.log | head_escaped + } + + rg -Fa " received signal " /test_output/gdb.log > /dev/null \ + && echo -e "Found signal in gdb.log$FAIL$(get_gdb_log_context)" >> /test_output/test_results.tsv + + dmesg -T > /test_output/dmesg.log + + # OOM in dmesg -- those are real + grep -q -F -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE' /test_output/dmesg.log \ + && echo -e "OOM in dmesg$FAIL$(head_escaped /test_output/dmesg.log)" >> /test_output/test_results.tsv \ + || echo -e "No OOM in dmesg$OK" >> /test_output/test_results.tsv +} + +function collect_query_and_trace_logs() +{ + for table in query_log trace_log + do + clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||: + done +} + +function collect_core_dumps() +{ + find . -type f -maxdepth 1 -name 'core.*' | while read core; do + zstd --threads=0 $core + mv $core.zst /test_output/ + done +} diff --git a/tests/ci/upgrade_check.py b/tests/ci/upgrade_check.py new file mode 100644 index 00000000000..83b6f9e299f --- /dev/null +++ b/tests/ci/upgrade_check.py @@ -0,0 +1,4 @@ +import stress_check + +if __name__ == "__main__": + stress_check.run_stress_test("clickhouse/upgrade-check") diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index a4a5a013c36..decf6ce0393 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -123,6 +123,7 @@ TRUSTED_CONTRIBUTORS = { "tonickkozlov", # Cloudflare "tylerhannan", # ClickHouse Employee "myrrc", # Mike Kot, DoubleCloud + "thevar1able", # ClickHouse Employee ] } diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c39c867fd5b..ce59ede8fd7 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -450,7 +450,7 @@ class FailureReason(enum.Enum): REPLICATED_DB = "replicated-database" S3_STORAGE = "s3-storage" BUILD = "not running for current build" - BACKWARD_INCOMPATIBLE = "test is backward incompatible" + NO_UPGRADE_CHECK = "not running for upgrade check" NO_PARALLEL_REPLICAS = "smth in not supported with parallel replicas" # UNKNOWN reasons @@ -774,35 +774,6 @@ class TestCase: else "" ) - # Check if test contains tag "no-backward-compatibility-check" and we should skip it - def check_backward_incompatible_tag(self) -> bool: - for tag in self.tags: - if tag.startswith("no-backward-compatibility-check"): - split = tag.split(":") - - # If version is not specified in tag, always skip this test. - if len(split) == 1: - return True - version_from_tag = split[1] - - # Check if extracted string from tag is a real ClickHouse version, if not - always skip test. - if re.match(VERSION_PATTERN, version_from_tag) is None: - return True - - server_version = str( - clickhouse_execute(args, "SELECT version()").decode() - ) - # If server version is less or equal from the version specified in tag, we should skip this test. - version_from_tag_split = list(map(int, version_from_tag.split("."))) - server_version_split = list(map(int, server_version.split("."))) - if ( - server_version_split[: len(version_from_tag_split)] - <= version_from_tag_split - ): - return True - - return False - # should skip test, should increment skipped_total, skip reason def should_skip_test(self, suite) -> Optional[FailureReason]: tags = self.tags @@ -853,10 +824,10 @@ class TestCase: elif tags and ("no-replicated-database" in tags) and args.replicated_database: return FailureReason.REPLICATED_DB - elif ( - args.backward_compatibility_check and self.check_backward_incompatible_tag() - ): - return FailureReason.BACKWARD_INCOMPATIBLE + # TODO: remove checking "no-upgrade-check" after 23.1 + elif args.upgrade_check and ( + "no-upgrade-check" in tags or "no-upgrade-check" in tags): + return FailureReason.NO_UPGRADE_CHECK elif tags and ("no-s3-storage" in tags) and args.s3_storage: return FailureReason.S3_STORAGE @@ -2468,9 +2439,9 @@ def parse_args(): ) group.add_argument( - "--backward-compatibility-check", + "--upgrade-check", action="store_true", - help="Run tests for further backward compatibility testing by ignoring all" + help="Run tests for further server upgrade testing by ignoring all" "drop queries in tests for collecting data from new version of server", ) parser.add_argument( @@ -2621,7 +2592,7 @@ if __name__ == "__main__": else: args.client_database = "default" - if args.backward_compatibility_check: + if args.upgrade_check: args.client += " --fake-drop" if args.client_option or args.secure: diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index cffd325e968..2066dedfa56 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -9,7 +9,8 @@ 10000 100000 10000 - false + + true 240000 1000000000000000 diff --git a/tests/integration/test_config_xml_full/test.py b/tests/integration/test_config_xml_full/test.py index ada3dc3f027..5f0679e5ef8 100644 --- a/tests/integration/test_config_xml_full/test.py +++ b/tests/integration/test_config_xml_full/test.py @@ -58,5 +58,19 @@ def test_xml_full_conf(): == "64999\n" ) + assert ( + node.query( + "select value from system.server_settings where name = 'max_connections'" + ) + == "4096\n" + ) + + assert ( + node.query( + "select changed from system.server_settings where name = 'max_connections'" + ) + == "1\n" + ) + finally: cluster.shutdown() diff --git a/tests/integration/test_config_xml_yaml_mix/test.py b/tests/integration/test_config_xml_yaml_mix/test.py index 373c42b2dea..9896d700ac1 100644 --- a/tests/integration/test_config_xml_yaml_mix/test.py +++ b/tests/integration/test_config_xml_yaml_mix/test.py @@ -64,6 +64,12 @@ def test_extra_yaml_mix(): ) == "64999\n" ) + assert ( + node.query( + "select value from system.server_settings where name = 'mark_cache_size'" + ) + == "8956\n" + ) finally: cluster.shutdown() diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 68c7cb96b71..33796336550 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -628,6 +628,27 @@ def test_table_override(started_cluster): assert_eq_with_retry(instance, query, expected) +def test_materialized_view(started_cluster): + cursor = pg_manager.get_db_cursor() + cursor.execute(f"DROP TABLE IF EXISTS test_table") + cursor.execute(f"CREATE TABLE test_table (key integer PRIMARY KEY, value integer)") + cursor.execute(f"INSERT INTO test_table SELECT 1, 2") + instance.query("DROP DATABASE IF EXISTS test_database") + instance.query( + "CREATE DATABASE test_database ENGINE = MaterializedPostgreSQL(postgres1) SETTINGS materialized_postgresql_tables_list='test_table'" + ) + check_tables_are_synchronized(instance, "test_table") + instance.query("DROP TABLE IF EXISTS mv") + instance.query( + "CREATE MATERIALIZED VIEW mv ENGINE=MergeTree ORDER BY tuple() POPULATE AS SELECT * FROM test_database.test_table" + ) + assert "1\t2" == instance.query("SELECT * FROM mv").strip() + cursor.execute(f"INSERT INTO test_table SELECT 3, 4") + check_tables_are_synchronized(instance, "test_table") + assert "1\t2\n3\t4" == instance.query("SELECT * FROM mv ORDER BY 1, 2").strip() + pg_manager.drop_materialized_db() + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_zero_copy_fetch/__init__.py b/tests/integration/test_zero_copy_fetch/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_zero_copy_fetch/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml b/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml new file mode 100644 index 00000000000..257ae0a355c --- /dev/null +++ b/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml @@ -0,0 +1,30 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + + + default + +
+ s3 + False + True +
+ + + + + + + true + + diff --git a/tests/integration/test_zero_copy_fetch/test.py b/tests/integration/test_zero_copy_fetch/test.py new file mode 100644 index 00000000000..f13eac5e9d1 --- /dev/null +++ b/tests/integration/test_zero_copy_fetch/test.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +import logging +import random +import string +import time + +import pytest +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml"], + with_minio=True, + with_zookeeper=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/storage_conf.xml"], + with_minio=True, + with_zookeeper=True, + ) + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def test_fetch_correct_volume(started_cluster): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query( + """ +CREATE TABLE test1 (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test1', 'r1') +PARTITION BY toMonday(EventDate) +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3'""" + ) + + node1.query( + "INSERT INTO test1 SELECT toDate('2023-01-01') + toIntervalDay(number), number + 1000 from system.numbers limit 20" + ) + + def get_part_to_disk(query_result): + part_to_disk = {} + for row in query_result.strip().split("\n"): + print(row) + disk, part = row.split("\t") + part_to_disk[part] = disk + return part_to_disk + + part_to_disk = get_part_to_disk( + node1.query( + "SELECT disk_name, name FROM system.parts where table = 'test1' and active" + ) + ) + for disk in part_to_disk.values(): + assert disk == "default" + + node1.query("ALTER TABLE test1 MOVE PARTITION '2022-12-26' TO DISK 's3'") + node1.query("ALTER TABLE test1 MOVE PARTITION '2023-01-02' TO DISK 's3'") + node1.query("ALTER TABLE test1 MOVE PARTITION '2023-01-09' TO DISK 's3'") + + part_to_disk = get_part_to_disk( + node1.query( + "SELECT disk_name, name FROM system.parts where table = 'test1' and active" + ) + ) + assert part_to_disk["20221226_0_0_0"] == "s3" + assert part_to_disk["20230102_0_0_0"] == "s3" + assert part_to_disk["20230109_0_0_0"] == "s3" + assert part_to_disk["20230116_0_0_0"] == "default" + + node2.query( + """ +CREATE TABLE test1 (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test1', 'r2') +PARTITION BY toMonday(EventDate) +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3'""" + ) + + node2.query("SYSTEM SYNC REPLICA test1") + + part_to_disk = get_part_to_disk( + node2.query( + "SELECT disk_name, name FROM system.parts where table = 'test1' and active" + ) + ) + assert part_to_disk["20221226_0_0_0"] == "s3" + assert part_to_disk["20230102_0_0_0"] == "s3" + assert part_to_disk["20230109_0_0_0"] == "s3" + assert part_to_disk["20230116_0_0_0"] == "default" diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml index dbeaf3ce6aa..403f94d8ac8 100644 --- a/tests/performance/aggregation_by_partitions.xml +++ b/tests/performance/aggregation_by_partitions.xml @@ -1,9 +1,8 @@ - - - - + 1 + 1 + 256 0 256 diff --git a/tests/queries/0_stateless/00061_merge_tree_alter.sql b/tests/queries/0_stateless/00061_merge_tree_alter.sql index ee5694518d9..2e46b1e16d6 100644 --- a/tests/queries/0_stateless/00061_merge_tree_alter.sql +++ b/tests/queries/0_stateless/00061_merge_tree_alter.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check DROP TABLE IF EXISTS alter_00061; set allow_deprecated_syntax_for_merge_tree=1; diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 1aa02864815..8b07d9abe35 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-parallel, no-s3-storage, no-backward-compatibility-check +# Tags: zookeeper, no-parallel, no-s3-storage, no-upgrade-check # Because REPLACE PARTITION does not forces immediate removal of replaced data parts from local filesystem # (it tries to do it as quick as possible, but it still performed in separate thread asynchronously) diff --git a/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql b/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql index e8d923389e5..9c02ac795ed 100644 --- a/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql +++ b/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql @@ -1,4 +1,4 @@ --- Tags: long, zookeeper, no-replicated-database, no-backward-compatibility-check +-- Tags: long, zookeeper, no-replicated-database, no-upgrade-check -- Tag no-replicated-database: Fails due to additional replicas or shards SET send_logs_level = 'fatal'; diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 7f5d5fabf12..72e0df5bfda 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -19,6 +19,7 @@ a --JSONType-- Object Array +Bool --JSONExtract-- -100 200 @@ -172,6 +173,7 @@ a --JSONType-- Object Array +Bool --JSONExtract-- -100 200 diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index dfae54dfd16..3314a5b762d 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -28,6 +28,7 @@ SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', -2); SELECT '--JSONType--'; SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}'); SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONType('{"a": true}', 'a'); SELECT '--JSONExtract--'; SELECT JSONExtractInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1); @@ -196,6 +197,7 @@ SELECT JSONKey('{"a": "hello", "b": [-100, 200.0, 300]}', -2); SELECT '--JSONType--'; SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}'); SELECT JSONType('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONType('{"a": true}', 'a'); SELECT '--JSONExtract--'; SELECT JSONExtractInt('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 1); diff --git a/tests/queries/0_stateless/00942_dataparts_500.sh b/tests/queries/0_stateless/00942_dataparts_500.sh index 7e1a7f15810..a6c3fcd4303 100755 --- a/tests/queries/0_stateless/00942_dataparts_500.sh +++ b/tests/queries/0_stateless/00942_dataparts_500.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-backward-compatibility-check +# Tags: no-upgrade-check # Test fix for issue #5066 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 5ccef802c0c..f143c97bdf4 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, zookeeper, no-parallel, no-backward-compatibility-check +# Tags: race, zookeeper, no-parallel, no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index 1f316b4b389..aec27792603 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-parallel, no-fasttest, no-backward-compatibility-check +# Tags: zookeeper, no-parallel, no-fasttest, no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh index bbe3a5a51c0..8c9efb75e96 100755 --- a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh +++ b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, zookeeper, no-backward-compatibility-check +# Tags: race, zookeeper, no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01191_rename_dictionary.sql b/tests/queries/0_stateless/01191_rename_dictionary.sql index ed9bc8af61b..8074e84f0ed 100644 --- a/tests/queries/0_stateless/01191_rename_dictionary.sql +++ b/tests/queries/0_stateless/01191_rename_dictionary.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel, no-backward-compatibility-check +-- Tags: no-parallel, no-upgrade-check DROP DATABASE IF EXISTS test_01191; CREATE DATABASE test_01191 ENGINE=Atomic; diff --git a/tests/queries/0_stateless/01221_system_settings.reference b/tests/queries/0_stateless/01221_system_settings.reference index 41f96bdceb5..399b3778b66 100644 --- a/tests/queries/0_stateless/01221_system_settings.reference +++ b/tests/queries/0_stateless/01221_system_settings.reference @@ -1,4 +1,4 @@ -send_timeout 300 0 Timeout for sending data to network, in seconds. If client needs to sent some data, but it did not able to send any bytes in this interval, exception is thrown. If you set this setting on client, the \'receive_timeout\' for the socket will be also set on the corresponding connection end on the server. \N \N 0 Seconds +send_timeout 300 0 Timeout for sending data to network, in seconds. If client needs to sent some data, but it did not able to send any bytes in this interval, exception is thrown. If you set this setting on client, the \'receive_timeout\' for the socket will be also set on the corresponding connection end on the server. \N \N 0 Seconds 300 storage_policy default 0 Name of storage disk policy \N \N 0 String 1 1 diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.reference b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.reference index b50519b9b3a..ea04f155f24 100644 --- a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.reference +++ b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_long.reference @@ -22,7 +22,7 @@ SELECT min(n) + 1, 1 + min(n), min(n) - 1, - 1 - min(n) + 1 - max(n) FROM ( SELECT number AS n @@ -42,7 +42,7 @@ SELECT max(n) + 1, 1 + max(n), max(n) - 1, - 1 - max(n) + 1 - min(n) FROM ( SELECT number AS n @@ -82,7 +82,7 @@ SELECT min(n) + -1, -1 + min(n), min(n) - -1, - -1 - min(n) + -1 - max(n) FROM ( SELECT number AS n @@ -102,7 +102,7 @@ SELECT max(n) + -1, -1 + max(n), max(n) - -1, - -1 - max(n) + -1 - min(n) FROM ( SELECT number AS n @@ -142,7 +142,7 @@ SELECT min(abs(2)) + 1, min(abs(2) + n), min(n - abs(2)), - 1 - min(abs(2)) + 1 - max(abs(2)) FROM ( SELECT number AS n @@ -162,7 +162,7 @@ SELECT max(abs(2)) + 1, max(abs(2) + n), max(n - abs(2)), - 1 - max(abs(2)) + 1 - min(abs(2)) FROM ( SELECT number AS n @@ -202,7 +202,7 @@ SELECT min(abs(n)) + 1, min(abs(n) + n), min(n - abs(n)), - 1 - min(abs(n)) + 1 - max(abs(n)) FROM ( SELECT number AS n @@ -222,7 +222,7 @@ SELECT max(abs(n)) + 1, max(abs(n) + n), max(n - abs(n)), - 1 - max(abs(n)) + 1 - min(abs(n)) FROM ( SELECT number AS n @@ -262,7 +262,7 @@ SELECT min(n * n) + 1, 1 + min(n * n), min(n * n) - 1, - 1 - min(n * n) + 1 - max(n * n) FROM ( SELECT number AS n @@ -282,7 +282,7 @@ SELECT max(n * n) + 1, 1 + max(n * n), max(n * n) - 1, - 1 - max(n * n) + 1 - min(n * n) FROM ( SELECT number AS n @@ -382,7 +382,7 @@ SELECT (min(n) + -1) + -1, (-1 + min(n)) + -1, (min(n) - -1) + -1, - (-1 - min(n)) + -1 + (-1 - max(n)) + -1 FROM ( SELECT number AS n @@ -402,7 +402,7 @@ SELECT (max(n) + -1) + -1, (-1 + max(n)) + -1, (max(n) - -1) + -1, - (-1 - max(n)) + -1 + (-1 - min(n)) + -1 FROM ( SELECT number AS n @@ -430,7 +430,7 @@ FROM SELECT number AS n FROM numbers(10) ) -SELECT (((min(n) + 1) + (1 + min(n))) + (min(n) - 1)) + (1 - min(n)) +SELECT (((min(n) + 1) + (1 + min(n))) + (min(n) - 1)) + (1 - max(n)) FROM ( SELECT number AS n @@ -442,7 +442,7 @@ FROM SELECT number AS n FROM numbers(10) ) -SELECT (((max(n) + 1) + (1 + max(n))) + (max(n) - 1)) + (1 - max(n)) +SELECT (((max(n) + 1) + (1 + max(n))) + (max(n) - 1)) + (1 - min(n)) FROM ( SELECT number AS n @@ -456,15 +456,15 @@ FROM ) 55 55 35 -35 90 90 22.5 inf -1 1 -1 1 +1 1 -1 -8 0 0 0 0.1111111111111111 -10 10 8 -8 +10 10 8 1 18 18 4.5 inf 35 35 55 -55 -90 -90 -22.5 -inf --1 -1 1 -1 +-1 -1 1 -10 -18 -18 -4.5 -inf -8 8 10 -10 +8 8 10 -1 0 0 -0 -0.1111111111111111 30 65 25 -10 40 90 22.5 5 @@ -474,15 +474,15 @@ FROM 4 18 4.5 0.5 55 90 0 -35 90 285 nan inf -1 0 0 1 +1 0 0 -8 0 0 nan 0.1111111111111111 -10 18 0 -8 +10 18 0 1 18 81 nan inf 295 295 275 -275 570 570 142.5 nan -1 1 -1 1 +1 1 -1 -80 0 0 0 nan -82 82 80 -80 +82 82 80 1 162 162 40.5 nan 65 65 45 -25 100 100 32.5 inf @@ -492,15 +492,15 @@ FROM 19 19 5.5 inf 25 25 45 -65 90 90 22.5 inf --2 -2 0 -2 +-2 -2 0 -11 0 0 0 0.1111111111111111 -7 7 9 -11 +7 7 9 -2 18 18 4.5 inf 110 inf -2 +-7 0.1111111111111111 -20 +29 inf -15444 68.62157087543459 diff --git a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh index f7615974237..f9a2ec8a34c 100755 --- a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh +++ b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, zookeeper, no-parallel, no-backward-compatibility-check +# Tags: long, zookeeper, no-parallel, no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.sql b/tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.sql index 8717c93b468..43c9fa43104 100644 --- a/tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.sql +++ b/tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.sql @@ -1,4 +1,4 @@ --- Tags: zookeeper, no-backward-compatibility-check +-- Tags: zookeeper, no-upgrade-check DROP TABLE IF EXISTS table_rename_with_ttl; diff --git a/tests/queries/0_stateless/01391_join_on_dict_crash.sql b/tests/queries/0_stateless/01391_join_on_dict_crash.sql index 13ebd080621..5321e03767f 100644 --- a/tests/queries/0_stateless/01391_join_on_dict_crash.sql +++ b/tests/queries/0_stateless/01391_join_on_dict_crash.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel, no-backward-compatibility-check +-- Tags: no-parallel, no-upgrade-check DROP DATABASE IF EXISTS db_01391; CREATE DATABASE db_01391; diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index 285e93a4f90..fea75e1439f 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check -- force data path with the user/pass in it set use_compact_format_in_distributed_parts_names=0; diff --git a/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh b/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh index 27de10ab16a..4a9b4beee5b 100755 --- a/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh +++ b/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-backward-compatibility-check +# Tags: no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql b/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql index 1dae8e7b383..b45a1974611 100644 --- a/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql +++ b/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql @@ -1,4 +1,4 @@ --- Tags: long, no-backward-compatibility-check +-- Tags: long, no-upgrade-check DROP TABLE IF EXISTS test_01640; DROP TABLE IF EXISTS restore_01640; diff --git a/tests/queries/0_stateless/01780_column_sparse_alter.sql b/tests/queries/0_stateless/01780_column_sparse_alter.sql index 925b81ea2c2..bc2f6f7c91f 100644 --- a/tests/queries/0_stateless/01780_column_sparse_alter.sql +++ b/tests/queries/0_stateless/01780_column_sparse_alter.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SET mutations_sync = 2; diff --git a/tests/queries/0_stateless/02022_storage_filelog_one_file.sh b/tests/queries/0_stateless/02022_storage_filelog_one_file.sh index 2f43423e13e..3abf5c52031 100755 --- a/tests/queries/0_stateless/02022_storage_filelog_one_file.sh +++ b/tests/queries/0_stateless/02022_storage_filelog_one_file.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-backward-compatibility-check +# Tags: no-upgrade-check set -eu diff --git a/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh b/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh index e4041b2d755..e0f0114d030 100755 --- a/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh +++ b/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-backward-compatibility-check +# Tags: no-upgrade-check set -eu diff --git a/tests/queries/0_stateless/02067_lost_part_s3.sql b/tests/queries/0_stateless/02067_lost_part_s3.sql index c4e69f68a5d..12afdcd4421 100644 --- a/tests/queries/0_stateless/02067_lost_part_s3.sql +++ b/tests/queries/0_stateless/02067_lost_part_s3.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check, no-fasttest +-- Tags: no-upgrade-check, no-fasttest DROP TABLE IF EXISTS partslost_0; DROP TABLE IF EXISTS partslost_1; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index bfda5e7afec..f77076bcd5c 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -977,6 +977,7 @@ CREATE TABLE system.settings `max` Nullable(String), `readonly` UInt8, `type` String, + `default` String, `alias_for` String ) ENGINE = SystemSettings diff --git a/tests/queries/0_stateless/02149_schema_inference_formats_with_schema.reference b/tests/queries/0_stateless/02149_schema_inference_formats_with_schema.reference index f3bfd9f1048..47391a77ee8 100644 --- a/tests/queries/0_stateless/02149_schema_inference_formats_with_schema.reference +++ b/tests/queries/0_stateless/02149_schema_inference_formats_with_schema.reference @@ -72,7 +72,7 @@ uint8 Nullable(UInt8) int16 Nullable(Int16) uint16 Nullable(UInt16) int32 Nullable(Int32) -uint32 Nullable(Int64) +uint32 Nullable(UInt32) int64 Nullable(Int64) uint64 Nullable(UInt64) 0 0 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh b/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh index 261c389c9f2..26646bd91a0 100755 --- a/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh +++ b/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-parallel, no-backward-compatibility-check +# Tags: no-fasttest, no-parallel, no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh index 8c28cdb57bd..f22b7796bd4 100755 --- a/tests/queries/0_stateless/02242_delete_user_race.sh +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, no-fasttest, no-parallel, no-backward-compatibility-check +# Tags: race, no-fasttest, no-parallel, no-upgrade-check # Test tries to reproduce a race between threads: # - deletes user diff --git a/tests/queries/0_stateless/02243_drop_user_grant_race.sh b/tests/queries/0_stateless/02243_drop_user_grant_race.sh index d36db47e562..e36be96aa02 100755 --- a/tests/queries/0_stateless/02243_drop_user_grant_race.sh +++ b/tests/queries/0_stateless/02243_drop_user_grant_race.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, no-fasttest, no-parallel, no-backward-compatibility-check +# Tags: race, no-fasttest, no-parallel, no-upgrade-check set -e diff --git a/tests/queries/0_stateless/02293_hashid.sql b/tests/queries/0_stateless/02293_hashid.sql index 9938154f169..06af0b5e1d8 100644 --- a/tests/queries/0_stateless/02293_hashid.sql +++ b/tests/queries/0_stateless/02293_hashid.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SET allow_experimental_hash_functions = 1; select number, hashid(number) from system.numbers limit 5; diff --git a/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql b/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql index 469476c82bf..8e0fb4a55a0 100644 --- a/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql +++ b/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SET max_bytes_in_join = '100', join_algorithm = 'auto'; diff --git a/tests/queries/0_stateless/02306_window_move_row_number_fix.sql b/tests/queries/0_stateless/02306_window_move_row_number_fix.sql index 5bc0c41b3ee..f73525f92be 100644 --- a/tests/queries/0_stateless/02306_window_move_row_number_fix.sql +++ b/tests/queries/0_stateless/02306_window_move_row_number_fix.sql @@ -1,2 +1,2 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SELECT nth_value(NULL, 1048577) OVER (Rows BETWEEN 1023 FOLLOWING AND UNBOUNDED FOLLOWING) diff --git a/tests/queries/0_stateless/02313_cross_join_dup_col_names.sql b/tests/queries/0_stateless/02313_cross_join_dup_col_names.sql index 44a4797ae3c..08e8843f763 100644 --- a/tests/queries/0_stateless/02313_cross_join_dup_col_names.sql +++ b/tests/queries/0_stateless/02313_cross_join_dup_col_names.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check -- https://github.com/ClickHouse/ClickHouse/issues/37561 diff --git a/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql index 47b47101a79..df20e5c42d4 100644 --- a/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql +++ b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SET join_algorithm = 'partial_merge'; diff --git a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql index 35f210be43d..cac7992e305 100644 --- a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql +++ b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql @@ -1,6 +1,3 @@ --- Tags: no-backward-compatibility-check --- TODO: remove no-backward-compatibility-check after new 22.6 release - SET cast_ipv4_ipv6_default_on_conversion_error = 1; DROP TABLE IF EXISTS ipv4_test; diff --git a/tests/queries/0_stateless/02316_const_string_intersact.sql b/tests/queries/0_stateless/02316_const_string_intersact.sql index 18af398aa5d..148d048952b 100644 --- a/tests/queries/0_stateless/02316_const_string_intersact.sql +++ b/tests/queries/0_stateless/02316_const_string_intersact.sql @@ -1,3 +1,3 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SELECT 'Play ClickHouse' InterSect SELECT 'Play ClickHouse' diff --git a/tests/queries/0_stateless/02320_mapped_array_witn_const_nullable.sql b/tests/queries/0_stateless/02320_mapped_array_witn_const_nullable.sql index 08651590c76..734c597051e 100644 --- a/tests/queries/0_stateless/02320_mapped_array_witn_const_nullable.sql +++ b/tests/queries/0_stateless/02320_mapped_array_witn_const_nullable.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check select arrayMap(x -> toNullable(1), range(number)) from numbers(3); select arrayFilter(x -> toNullable(1), range(number)) from numbers(3); diff --git a/tests/queries/0_stateless/02332_dist_insert_send_logs_level.sh b/tests/queries/0_stateless/02332_dist_insert_send_logs_level.sh index 653cb25172a..503b94be715 100755 --- a/tests/queries/0_stateless/02332_dist_insert_send_logs_level.sh +++ b/tests/queries/0_stateless/02332_dist_insert_send_logs_level.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-backward-compatibility-check +# Tags: no-upgrade-check CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=trace CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql index eb395e5ec41..07f705acd84 100644 --- a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql +++ b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check -- Regression for PartialSortingTransform optimization that requires at least 1500 rows. SELECT * FROM (SELECT * FROM (SELECT 0 a, toNullable(number) b, toString(number) c FROM numbers(1e6)) ORDER BY a DESC, b DESC, c LIMIT 1500) limit 10; diff --git a/tests/queries/0_stateless/02354_annoy.sh b/tests/queries/0_stateless/02354_annoy.sh index 7b49a338955..87258debf0f 100755 --- a/tests/queries/0_stateless/02354_annoy.sh +++ b/tests/queries/0_stateless/02354_annoy.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-ubsan, no-cpu-aarch64, no-backward-compatibility-check +# Tags: no-fasttest, no-ubsan, no-cpu-aarch64, no-upgrade-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02363_mapupdate_improve.sql b/tests/queries/0_stateless/02363_mapupdate_improve.sql index 6b7723cc9b4..b4a4b8e5d91 100644 --- a/tests/queries/0_stateless/02363_mapupdate_improve.sql +++ b/tests/queries/0_stateless/02363_mapupdate_improve.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check DROP TABLE IF EXISTS map_test; CREATE TABLE map_test(`tags` Map(String, String)) ENGINE = MergeTree PRIMARY KEY tags ORDER BY tags SETTINGS index_granularity = 8192; INSERT INTO map_test (tags) VALUES (map('fruit','apple','color','red')); diff --git a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql index 9d6950051f0..cf9f2971cb0 100644 --- a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql +++ b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check DROP TABLE IF EXISTS test_table; CREATE TABLE test_table diff --git a/tests/queries/0_stateless/02366_with_fill_date.sql b/tests/queries/0_stateless/02366_with_fill_date.sql index 64e23b845f8..4d41facf423 100644 --- a/tests/queries/0_stateless/02366_with_fill_date.sql +++ b/tests/queries/0_stateless/02366_with_fill_date.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-upgrade-check SELECT toDate('2022-02-01') AS d1 FROM numbers(18) AS number diff --git a/tests/queries/0_stateless/02377_extend_protocol_with_query_parameters.reference b/tests/queries/0_stateless/02377_extend_protocol_with_query_parameters.reference index 1da5cd0b7b3..8b384f5e0d1 100644 --- a/tests/queries/0_stateless/02377_extend_protocol_with_query_parameters.reference +++ b/tests/queries/0_stateless/02377_extend_protocol_with_query_parameters.reference @@ -9,8 +9,8 @@ UInt64 String DateTime Map(UUID, Array(Float32)) 1 _CAST(42, \'Int64\') Int64 _CAST([1, 2, 3], \'Array(UInt8)\') Array(UInt8) -_CAST(((\'abc\', 22), (\'def\', 33)), \'Map(String, UInt8)\') Map(String, UInt8) +_CAST([(\'abc\', 22), (\'def\', 33)], \'Map(String, UInt8)\') Map(String, UInt8) _CAST([[4, 5, 6], [7], [8, 9]], \'Array(Array(UInt8))\') Array(Array(UInt8)) -_CAST(((10, [11, 12]), (13, [14, 15])), \'Map(UInt8, Array(UInt8))\') Map(UInt8, Array(UInt8)) -_CAST(((\'ghj\', ((\'klm\', [16, 17]))), (\'nop\', ((\'rst\', [18])))), \'Map(String, Map(String, Array(UInt8)))\') Map(String, Map(String, Array(UInt8))) +_CAST([(10, [11, 12]), (13, [14, 15])], \'Map(UInt8, Array(UInt8))\') Map(UInt8, Array(UInt8)) +_CAST([(\'ghj\', [(\'klm\', [16, 17])]), (\'nop\', [(\'rst\', [18])])], \'Map(String, Map(String, Array(UInt8)))\') Map(String, Map(String, Array(UInt8))) a Int8 diff --git a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql index 7eaa46242ae..842e22ba87d 100644 --- a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql +++ b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check, no-random-merge-tree-settings +-- Tags: no-upgrade-check, no-random-merge-tree-settings drop table if exists test_02381; create table test_02381(a UInt64, b UInt64) ENGINE = MergeTree order by (a, b) SETTINGS compress_marks=false, compress_primary_key=false; diff --git a/tests/queries/0_stateless/02397_system_parts_race_condition_drop_rm.sh b/tests/queries/0_stateless/02397_system_parts_race_condition_drop_rm.sh index 2372d30497e..548179b94c9 100755 --- a/tests/queries/0_stateless/02397_system_parts_race_condition_drop_rm.sh +++ b/tests/queries/0_stateless/02397_system_parts_race_condition_drop_rm.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, zookeeper, no-parallel, no-backward-compatibility-check, disabled +# Tags: race, zookeeper, no-parallel, no-upgrade-check, disabled CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02429_low_cardinality_trash.sh b/tests/queries/0_stateless/02429_low_cardinality_trash.sh index 258f02b4bb6..91618cb2796 100755 --- a/tests/queries/0_stateless/02429_low_cardinality_trash.sh +++ b/tests/queries/0_stateless/02429_low_cardinality_trash.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-backward-compatibility-check +# Tags: long, no-upgrade-check CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh index 11ca3f43d8f..abcf1bf4c5b 100755 --- a/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh +++ b/tests/queries/0_stateless/02450_kill_distributed_query_deadlock.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-backward-compatibility-check +# Tags: long, no-upgrade-check CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference index 4f9430ef608..43282d09bab 100644 --- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference +++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference @@ -1 +1,2 @@ 4 2 +100 99 diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql index e3e508e17be..5eba14ea528 100644 --- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql +++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql @@ -12,3 +12,5 @@ INSERT INTO test_table VALUES (1, 1); INSERT INTO test_table VALUES (1, 1); SELECT sum((2 * id) as func), func FROM test_table GROUP BY id; + +SELECT max(100-number), min(100-number) FROM numbers(2); diff --git a/tests/queries/0_stateless/02661_window_ntile.reference b/tests/queries/0_stateless/02560_window_ntile.reference similarity index 100% rename from tests/queries/0_stateless/02661_window_ntile.reference rename to tests/queries/0_stateless/02560_window_ntile.reference diff --git a/tests/queries/0_stateless/02661_window_ntile.sql b/tests/queries/0_stateless/02560_window_ntile.sql similarity index 100% rename from tests/queries/0_stateless/02661_window_ntile.sql rename to tests/queries/0_stateless/02560_window_ntile.sql diff --git a/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.reference b/tests/queries/0_stateless/02561_sorting_constants_and_distinct_crash.reference similarity index 100% rename from tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.reference rename to tests/queries/0_stateless/02561_sorting_constants_and_distinct_crash.reference diff --git a/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql b/tests/queries/0_stateless/02561_sorting_constants_and_distinct_crash.sql similarity index 100% rename from tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql rename to tests/queries/0_stateless/02561_sorting_constants_and_distinct_crash.sql diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02562_regexp_extract.reference similarity index 100% rename from tests/queries/0_stateless/02665_regexp_extract.reference rename to tests/queries/0_stateless/02562_regexp_extract.reference diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02562_regexp_extract.sql similarity index 100% rename from tests/queries/0_stateless/02665_regexp_extract.sql rename to tests/queries/0_stateless/02562_regexp_extract.sql diff --git a/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.reference b/tests/queries/0_stateless/02563_progress_when_no_rows_from_prewhere.reference similarity index 100% rename from tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.reference rename to tests/queries/0_stateless/02563_progress_when_no_rows_from_prewhere.reference diff --git a/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.sh b/tests/queries/0_stateless/02563_progress_when_no_rows_from_prewhere.sh similarity index 100% rename from tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.sh rename to tests/queries/0_stateless/02563_progress_when_no_rows_from_prewhere.sh diff --git a/tests/queries/0_stateless/02666_read_in_order_final_desc.reference b/tests/queries/0_stateless/02564_read_in_order_final_desc.reference similarity index 100% rename from tests/queries/0_stateless/02666_read_in_order_final_desc.reference rename to tests/queries/0_stateless/02564_read_in_order_final_desc.reference diff --git a/tests/queries/0_stateless/02666_read_in_order_final_desc.sql b/tests/queries/0_stateless/02564_read_in_order_final_desc.sql similarity index 100% rename from tests/queries/0_stateless/02666_read_in_order_final_desc.sql rename to tests/queries/0_stateless/02564_read_in_order_final_desc.sql diff --git a/tests/queries/0_stateless/02565_analyzer_limit_settings.reference b/tests/queries/0_stateless/02565_analyzer_limit_settings.reference new file mode 100644 index 00000000000..6f23097612e --- /dev/null +++ b/tests/queries/0_stateless/02565_analyzer_limit_settings.reference @@ -0,0 +1,70 @@ +-- { echoOn } +SET limit = 0; +SELECT * FROM numbers(10); +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +SELECT * FROM numbers(10) SETTINGS limit=5, offset=2; +2 +3 +4 +5 +6 +SELECT count(*) FROM (SELECT * FROM numbers(10)); +10 +SELECT count(*) FROM (SELECT * FROM numbers(10) SETTINGS limit=5); +5 +SELECT count(*) FROM (SELECT * FROM numbers(10)) SETTINGS limit=5; +10 +SELECT count(*) FROM view(SELECT * FROM numbers(10)); +10 +SELECT count(*) FROM view(SELECT * FROM numbers(10) SETTINGS limit=5); +5 +SELECT count(*) FROM view(SELECT * FROM numbers(10)) SETTINGS limit=5; +10 +SET limit = 3; +SELECT * FROM numbers(10); +0 +1 +2 +SELECT * FROM numbers(10) SETTINGS limit=5, offset=2; +2 +3 +4 +5 +6 +SELECT count(*) FROM (SELECT * FROM numbers(10)); +10 +SELECT count(*) FROM (SELECT * FROM numbers(10) SETTINGS limit=5); +5 +SELECT count(*) FROM (SELECT * FROM numbers(10)) SETTINGS limit=5; +10 +SELECT count(*) FROM view(SELECT * FROM numbers(10)); +10 +SELECT count(*) FROM view(SELECT * FROM numbers(10) SETTINGS limit=5); +5 +SELECT count(*) FROM view(SELECT * FROM numbers(10)) SETTINGS limit=5; +10 +SET limit = 4; +SET offset = 1; +SELECT * FROM numbers(10); +1 +2 +3 +4 +SELECT * FROM numbers(10) LIMIT 3 OFFSET 2; +3 +4 +5 +SELECT * FROM numbers(10) LIMIT 5 OFFSET 2; +3 +4 +5 +6 diff --git a/tests/queries/0_stateless/02565_analyzer_limit_settings.sql b/tests/queries/0_stateless/02565_analyzer_limit_settings.sql new file mode 100644 index 00000000000..7c02c2d0d20 --- /dev/null +++ b/tests/queries/0_stateless/02565_analyzer_limit_settings.sql @@ -0,0 +1,30 @@ +SET allow_experimental_analyzer = 1; + +-- { echoOn } +SET limit = 0; + +SELECT * FROM numbers(10); +SELECT * FROM numbers(10) SETTINGS limit=5, offset=2; +SELECT count(*) FROM (SELECT * FROM numbers(10)); +SELECT count(*) FROM (SELECT * FROM numbers(10) SETTINGS limit=5); +SELECT count(*) FROM (SELECT * FROM numbers(10)) SETTINGS limit=5; +SELECT count(*) FROM view(SELECT * FROM numbers(10)); +SELECT count(*) FROM view(SELECT * FROM numbers(10) SETTINGS limit=5); +SELECT count(*) FROM view(SELECT * FROM numbers(10)) SETTINGS limit=5; + +SET limit = 3; +SELECT * FROM numbers(10); +SELECT * FROM numbers(10) SETTINGS limit=5, offset=2; +SELECT count(*) FROM (SELECT * FROM numbers(10)); +SELECT count(*) FROM (SELECT * FROM numbers(10) SETTINGS limit=5); +SELECT count(*) FROM (SELECT * FROM numbers(10)) SETTINGS limit=5; +SELECT count(*) FROM view(SELECT * FROM numbers(10)); +SELECT count(*) FROM view(SELECT * FROM numbers(10) SETTINGS limit=5); +SELECT count(*) FROM view(SELECT * FROM numbers(10)) SETTINGS limit=5; + +SET limit = 4; +SET offset = 1; +SELECT * FROM numbers(10); +SELECT * FROM numbers(10) LIMIT 3 OFFSET 2; +SELECT * FROM numbers(10) LIMIT 5 OFFSET 2; +-- { echoOff } diff --git a/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.reference b/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.reference new file mode 100644 index 00000000000..466e80931e5 --- /dev/null +++ b/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.reference @@ -0,0 +1,14 @@ +limit 0 +limit 1 +limit 2 +limit 3 +limit 4 +offset 5 +offset 6 +offset 7 +offset 8 +offset 9 +limit w/ GROUP BY 4 4 +limit w/ GROUP BY 4 3 +limit/offset w/ GROUP BY 4 2 +limit/offset w/ GROUP BY 4 1 diff --git a/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql b/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql new file mode 100644 index 00000000000..1624344b5a9 --- /dev/null +++ b/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql @@ -0,0 +1,34 @@ +-- Tags: distributed + +SET allow_experimental_analyzer = 1; + +SELECT 'limit', * FROM remote('127.1', view(SELECT * FROM numbers(10))) SETTINGS limit=5; +SELECT 'offset', * FROM remote('127.1', view(SELECT * FROM numbers(10))) SETTINGS offset=5; + +SELECT + 'limit w/ GROUP BY', + count(), + number +FROM remote('127.{1,2}', view( + SELECT intDiv(number, 2) AS number + FROM numbers(10) +)) +GROUP BY number +ORDER BY + count() ASC, + number DESC +SETTINGS limit=2; + +SELECT + 'limit/offset w/ GROUP BY', + count(), + number +FROM remote('127.{1,2}', view( + SELECT intDiv(number, 2) AS number + FROM numbers(10) +)) +GROUP BY number +ORDER BY + count() ASC, + number DESC +SETTINGS limit=2, offset=2; diff --git a/tests/queries/0_stateless/02667_and_consistency.reference b/tests/queries/0_stateless/02567_and_consistency.reference similarity index 100% rename from tests/queries/0_stateless/02667_and_consistency.reference rename to tests/queries/0_stateless/02567_and_consistency.reference diff --git a/tests/queries/0_stateless/02667_and_consistency.sql b/tests/queries/0_stateless/02567_and_consistency.sql similarity index 100% rename from tests/queries/0_stateless/02667_and_consistency.sql rename to tests/queries/0_stateless/02567_and_consistency.sql diff --git a/tests/queries/0_stateless/02667_array_map_const_low_cardinality.reference b/tests/queries/0_stateless/02568_array_map_const_low_cardinality.reference similarity index 100% rename from tests/queries/0_stateless/02667_array_map_const_low_cardinality.reference rename to tests/queries/0_stateless/02568_array_map_const_low_cardinality.reference diff --git a/tests/queries/0_stateless/02667_array_map_const_low_cardinality.sql b/tests/queries/0_stateless/02568_array_map_const_low_cardinality.sql similarity index 100% rename from tests/queries/0_stateless/02667_array_map_const_low_cardinality.sql rename to tests/queries/0_stateless/02568_array_map_const_low_cardinality.sql diff --git a/tests/queries/0_stateless/02667_json_array_length.reference b/tests/queries/0_stateless/02568_json_array_length.reference similarity index 100% rename from tests/queries/0_stateless/02667_json_array_length.reference rename to tests/queries/0_stateless/02568_json_array_length.reference diff --git a/tests/queries/0_stateless/02667_json_array_length.sql b/tests/queries/0_stateless/02568_json_array_length.sql similarity index 100% rename from tests/queries/0_stateless/02667_json_array_length.sql rename to tests/queries/0_stateless/02568_json_array_length.sql diff --git a/tests/queries/0_stateless/02667_order_by_aggregation_result.reference b/tests/queries/0_stateless/02569_order_by_aggregation_result.reference similarity index 100% rename from tests/queries/0_stateless/02667_order_by_aggregation_result.reference rename to tests/queries/0_stateless/02569_order_by_aggregation_result.reference diff --git a/tests/queries/0_stateless/02667_order_by_aggregation_result.sql b/tests/queries/0_stateless/02569_order_by_aggregation_result.sql similarity index 100% rename from tests/queries/0_stateless/02667_order_by_aggregation_result.sql rename to tests/queries/0_stateless/02569_order_by_aggregation_result.sql diff --git a/tests/queries/0_stateless/02668_fallback_from_async_insert.reference b/tests/queries/0_stateless/02570_fallback_from_async_insert.reference similarity index 100% rename from tests/queries/0_stateless/02668_fallback_from_async_insert.reference rename to tests/queries/0_stateless/02570_fallback_from_async_insert.reference diff --git a/tests/queries/0_stateless/02668_fallback_from_async_insert.sh b/tests/queries/0_stateless/02570_fallback_from_async_insert.sh similarity index 100% rename from tests/queries/0_stateless/02668_fallback_from_async_insert.sh rename to tests/queries/0_stateless/02570_fallback_from_async_insert.sh diff --git a/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.reference b/tests/queries/0_stateless/02571_local_desc_abort_on_twitter_json.reference similarity index 100% rename from tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.reference rename to tests/queries/0_stateless/02571_local_desc_abort_on_twitter_json.reference diff --git a/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.sh b/tests/queries/0_stateless/02571_local_desc_abort_on_twitter_json.sh similarity index 100% rename from tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.sh rename to tests/queries/0_stateless/02571_local_desc_abort_on_twitter_json.sh diff --git a/tests/queries/0_stateless/02670_max_intersections.reference b/tests/queries/0_stateless/02572_max_intersections.reference similarity index 100% rename from tests/queries/0_stateless/02670_max_intersections.reference rename to tests/queries/0_stateless/02572_max_intersections.reference diff --git a/tests/queries/0_stateless/02670_max_intersections.sql b/tests/queries/0_stateless/02572_max_intersections.sql similarity index 100% rename from tests/queries/0_stateless/02670_max_intersections.sql rename to tests/queries/0_stateless/02572_max_intersections.sql diff --git a/tests/queries/0_stateless/02671_quantile_fuse_msan.reference b/tests/queries/0_stateless/02573_quantile_fuse_msan.reference similarity index 100% rename from tests/queries/0_stateless/02671_quantile_fuse_msan.reference rename to tests/queries/0_stateless/02573_quantile_fuse_msan.reference diff --git a/tests/queries/0_stateless/02671_quantile_fuse_msan.sql b/tests/queries/0_stateless/02573_quantile_fuse_msan.sql similarity index 100% rename from tests/queries/0_stateless/02671_quantile_fuse_msan.sql rename to tests/queries/0_stateless/02573_quantile_fuse_msan.sql diff --git a/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.reference b/tests/queries/0_stateless/02574_suspicious_low_cardinality_msan.reference similarity index 100% rename from tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.reference rename to tests/queries/0_stateless/02574_suspicious_low_cardinality_msan.reference diff --git a/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.sql b/tests/queries/0_stateless/02574_suspicious_low_cardinality_msan.sql similarity index 100% rename from tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.sql rename to tests/queries/0_stateless/02574_suspicious_low_cardinality_msan.sql diff --git a/tests/queries/0_stateless/02673_map_hashing_msan.reference b/tests/queries/0_stateless/02575_map_hashing_msan.reference similarity index 100% rename from tests/queries/0_stateless/02673_map_hashing_msan.reference rename to tests/queries/0_stateless/02575_map_hashing_msan.reference diff --git a/tests/queries/0_stateless/02673_map_hashing_msan.sql b/tests/queries/0_stateless/02575_map_hashing_msan.sql similarity index 100% rename from tests/queries/0_stateless/02673_map_hashing_msan.sql rename to tests/queries/0_stateless/02575_map_hashing_msan.sql diff --git a/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.reference b/tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.reference similarity index 100% rename from tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.reference rename to tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.reference diff --git a/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.sql b/tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.sql similarity index 100% rename from tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.sql rename to tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.sql diff --git a/tests/queries/0_stateless/02660_rewrite_array_exists_to_has.reference b/tests/queries/0_stateless/02576_rewrite_array_exists_to_has.reference similarity index 100% rename from tests/queries/0_stateless/02660_rewrite_array_exists_to_has.reference rename to tests/queries/0_stateless/02576_rewrite_array_exists_to_has.reference diff --git a/tests/queries/0_stateless/02660_rewrite_array_exists_to_has.sql b/tests/queries/0_stateless/02576_rewrite_array_exists_to_has.sql similarity index 100% rename from tests/queries/0_stateless/02660_rewrite_array_exists_to_has.sql rename to tests/queries/0_stateless/02576_rewrite_array_exists_to_has.sql diff --git a/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.reference b/tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.reference similarity index 100% rename from tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.reference rename to tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.reference diff --git a/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.sql b/tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.sql similarity index 100% rename from tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.sql rename to tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.sql diff --git a/tests/queries/0_stateless/02661_keepermap_delete_update.reference b/tests/queries/0_stateless/02577_keepermap_delete_update.reference similarity index 100% rename from tests/queries/0_stateless/02661_keepermap_delete_update.reference rename to tests/queries/0_stateless/02577_keepermap_delete_update.reference diff --git a/tests/queries/0_stateless/02661_keepermap_delete_update.sql b/tests/queries/0_stateless/02577_keepermap_delete_update.sql similarity index 100% rename from tests/queries/0_stateless/02661_keepermap_delete_update.sql rename to tests/queries/0_stateless/02577_keepermap_delete_update.sql diff --git a/tests/queries/0_stateless/25338_ipv4_codec_t64.reference b/tests/queries/0_stateless/02578_ipv4_codec_t64.reference similarity index 100% rename from tests/queries/0_stateless/25338_ipv4_codec_t64.reference rename to tests/queries/0_stateless/02578_ipv4_codec_t64.reference diff --git a/tests/queries/0_stateless/25338_ipv4_codec_t64.sql b/tests/queries/0_stateless/02578_ipv4_codec_t64.sql similarity index 100% rename from tests/queries/0_stateless/25338_ipv4_codec_t64.sql rename to tests/queries/0_stateless/02578_ipv4_codec_t64.sql diff --git a/tests/queries/0_stateless/02661_parameterized_rename_queries.reference b/tests/queries/0_stateless/02578_parameterized_rename_queries.reference similarity index 100% rename from tests/queries/0_stateless/02661_parameterized_rename_queries.reference rename to tests/queries/0_stateless/02578_parameterized_rename_queries.reference diff --git a/tests/queries/0_stateless/02661_parameterized_rename_queries.sql b/tests/queries/0_stateless/02578_parameterized_rename_queries.sql similarity index 100% rename from tests/queries/0_stateless/02661_parameterized_rename_queries.sql rename to tests/queries/0_stateless/02578_parameterized_rename_queries.sql diff --git a/tests/queries/0_stateless/25339_fill_empty_chunk.reference b/tests/queries/0_stateless/02579_fill_empty_chunk.reference similarity index 100% rename from tests/queries/0_stateless/25339_fill_empty_chunk.reference rename to tests/queries/0_stateless/02579_fill_empty_chunk.reference diff --git a/tests/queries/0_stateless/25339_fill_empty_chunk.sql b/tests/queries/0_stateless/02579_fill_empty_chunk.sql similarity index 100% rename from tests/queries/0_stateless/25339_fill_empty_chunk.sql rename to tests/queries/0_stateless/02579_fill_empty_chunk.sql diff --git a/tests/queries/0_stateless/02661_parameterized_replace.reference b/tests/queries/0_stateless/02579_parameterized_replace.reference similarity index 100% rename from tests/queries/0_stateless/02661_parameterized_replace.reference rename to tests/queries/0_stateless/02579_parameterized_replace.reference diff --git a/tests/queries/0_stateless/02661_parameterized_replace.sql b/tests/queries/0_stateless/02579_parameterized_replace.sql similarity index 100% rename from tests/queries/0_stateless/02661_parameterized_replace.sql rename to tests/queries/0_stateless/02579_parameterized_replace.sql diff --git a/tests/queries/0_stateless/25339_like_substring_search_bug.reference b/tests/queries/0_stateless/02580_like_substring_search_bug.reference similarity index 100% rename from tests/queries/0_stateless/25339_like_substring_search_bug.reference rename to tests/queries/0_stateless/02580_like_substring_search_bug.reference diff --git a/tests/queries/0_stateless/25339_like_substring_search_bug.sql b/tests/queries/0_stateless/02580_like_substring_search_bug.sql similarity index 100% rename from tests/queries/0_stateless/25339_like_substring_search_bug.sql rename to tests/queries/0_stateless/02580_like_substring_search_bug.sql diff --git a/tests/queries/0_stateless/02670_map_literal_cast.reference b/tests/queries/0_stateless/02670_map_literal_cast.reference new file mode 100644 index 00000000000..9f4e9fd8d08 --- /dev/null +++ b/tests/queries/0_stateless/02670_map_literal_cast.reference @@ -0,0 +1,4 @@ +{'a':1,'b':2} +{'abc':22,'def':33} +{10:[11,12],13:[14,15]} +{'ghj':{'klm':[16,17]},'nop':{'rst':[18]}} diff --git a/tests/queries/0_stateless/02670_map_literal_cast.sql b/tests/queries/0_stateless/02670_map_literal_cast.sql new file mode 100644 index 00000000000..23bb880889b --- /dev/null +++ b/tests/queries/0_stateless/02670_map_literal_cast.sql @@ -0,0 +1,9 @@ +SELECT CAST([('a', 1), ('b', 2)], 'Map(String, UInt8)'); +SELECT CAST([('abc', 22), ('def', 33)], 'Map(String, UInt8)'); +SELECT CAST([(10, [11, 12]), (13, [14, 15])], 'Map(UInt8, Array(UInt8))'); +SELECT CAST([('ghj', [('klm', [16, 17])]), ('nop', [('rst', [18])])], 'Map(String, Map(String, Array(UInt8)))'); + +SELECT CAST((('a', 1), ('b', 2)), 'Map(String, UInt8)'); -- { serverError TYPE_MISMATCH } +SELECT CAST((('abc', 22), ('def', 33)), 'Map(String, UInt8)'); -- { serverError TYPE_MISMATCH } +SELECT CAST(((10, [11, 12]), (13, [14, 15])), 'Map(UInt8, Array(UInt8))'); -- { serverError TYPE_MISMATCH } +SELECT CAST((('ghj', (('klm', [16, 17]))), ('nop', (('rst', [18])))), 'Map(String, Map(String, Array(UInt8)))'); -- { serverError TYPE_MISMATCH } diff --git a/tests/queries/0_stateless/02674_range_ipv4.reference b/tests/queries/0_stateless/02674_range_ipv4.reference new file mode 100644 index 00000000000..76fc0c45bd3 --- /dev/null +++ b/tests/queries/0_stateless/02674_range_ipv4.reference @@ -0,0 +1,3 @@ +[2887712768,2887712769,2887712770,2887712771,2887712772,2887712773,2887712774,2887712775,2887712776,2887712777] +[2887712768,2887712769,2887712770,2887712771,2887712772,2887712773,2887712774,2887712775,2887712776,2887712777] +[2887712768,2887712769,2887712770,2887712771,2887712772,2887712773,2887712774,2887712775,2887712776,2887712777] diff --git a/tests/queries/0_stateless/02674_range_ipv4.sql b/tests/queries/0_stateless/02674_range_ipv4.sql new file mode 100644 index 00000000000..1241b727014 --- /dev/null +++ b/tests/queries/0_stateless/02674_range_ipv4.sql @@ -0,0 +1,3 @@ +SELECT range(toIPv4('172.31.0.0'), toIPv4('172.31.0.10')); +SELECT range(2887712768, toIPv4('172.31.0.10')); +SELECT range(toIPv4('172.31.0.0'), 2887712778); diff --git a/tests/queries/0_stateless/25337_width_bucket.reference b/tests/queries/0_stateless/25337_width_bucket.reference new file mode 100644 index 00000000000..aa0e8f2612c --- /dev/null +++ b/tests/queries/0_stateless/25337_width_bucket.reference @@ -0,0 +1,156 @@ +-7.6 -10 0 4 1 +-6 -5 -1 2 0 +0 0 10 4 1 +1 3 0 1 0 +3 -100 200 10 4 +3 0 10 3 1 +4.333 1 11 3 1 +4.34 1 11 3 2 +---------- +0 +11 +---------- +-7 -10 0 4 2 +-6 -5 -1 2 0 +0 0 10 4 1 +1 3 0 1 0 +3 -100 200 10 4 +3 0 10 3 1 +4 1 11 3 1 +4 1 11 3 1 +---------- +249 65526 0 4 0 +250 65531 4294967295 2 0 +0 0 10 4 1 +1 3 0 1 0 +3 65436 200 10 0 +3 0 10 3 1 +4 1 11 3 1 +4 1 11 3 1 +---------- +---------- +UInt16 +UInt32 +UInt64 +UInt64 +---------- +5 +3 +1 +0 +4 +1 +1 +1 +0 +0 +0 +0 +1 +1 +1 +1 +0 +0 +1 +0 +1 +1 +1 +1 +1 +0 +1 +0 +10 +3 +4 +4 +5 +3 +5 +2 +11 +4 +4 +4 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +2 +0 +1 +0 +3 +3 +3 +3 +10 +10 +9 +0 +4 +9 +8 +8 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +1 +0 +0 +0 +2 +0 +1 +0 +12 +4 +4 +4 +14 +14 +14 +14 +14 +14 +14 +14 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/25337_width_bucket.sql b/tests/queries/0_stateless/25337_width_bucket.sql new file mode 100644 index 00000000000..b375623246a --- /dev/null +++ b/tests/queries/0_stateless/25337_width_bucket.sql @@ -0,0 +1,62 @@ +CREATE TABLE mytable +( + operand Float64, + low Float64, + high Float64, + count UInt64, + PRIMARY KEY (operand, low, high, count) +) ENGINE = MergeTree(); + +INSERT INTO mytable VALUES (3, -100, 200, 10), (0, 0, 10, 4), (3, 0, 10, 3), (4.333, 1, 11, 3), (4.34, 1, 11, 3), (-7.6, -10, 0, 4), (-6, -5, -1, 2), (1, 3, 0, 1), (3, 2, 5, 0); + +SELECT operand, low, high, count, WIDTH_BUCKET(operand, low, high, count) FROM mytable WHERE count != 0; +SELECT '----------'; +-- zero is not valid for count +SELECT operand, low, high, count, WIDTH_BUCKET(operand, low, high, count) FROM mytable WHERE count = 0; -- { serverError BAD_ARGUMENTS } +-- operand, low and high cannot be NaN +SELECT WIDTH_BUCKET(0, 10, NaN, 10); -- { serverError BAD_ARGUMENTS } +SELECT WIDTH_BUCKET(NaN, 0, 10, 10); -- { serverError BAD_ARGUMENTS } +SELECT WIDTH_BUCKET(0, NaN, 10, 10); -- { serverError BAD_ARGUMENTS } +-- low and high cannot be Inf +SELECT WIDTH_BUCKET(1, -Inf, 10, 10); -- { serverError BAD_ARGUMENTS } +-- low and high cannot be Inf +SELECT WIDTH_BUCKET(1, 0, Inf, 10); -- { serverError BAD_ARGUMENTS } +-- operand can be Inf +SELECT WIDTH_BUCKET(-Inf, 0, 10, 10); +SELECT WIDTH_BUCKET(Inf, 0, 10, 10); +SELECT '----------'; +-- IntXX types +SELECT toInt64(operand) AS operand, toInt32(low) AS low, toInt16(high) AS high, count, WIDTH_BUCKET(operand, low, high, count) FROM mytable WHERE count != 0; +SELECT '----------'; +-- UIntXX types +SELECT toUInt8(toInt8(operand)) AS operand, toUInt16(toInt16(low)) AS low, toUInt32(toInt32(high)) AS high, count, WIDTH_BUCKET(operand, low, high, count) FROM mytable WHERE count != 0; +SELECT '----------'; +SELECT WIDTH_BUCKET(1, 2, 3, -1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT WIDTH_BUCKET(1, 2, 3, 1.3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT WIDTH_BUCKET('a', 1, 2, 3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT WIDTH_BUCKET(1, toUInt128(42), 2, 3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT WIDTH_BUCKET(1, 2, toInt128(42), 3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT WIDTH_BUCKET(1, 2, 3, toInt256(42)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT '----------'; +-- Return type checks +SELECT toTypeName(WIDTH_BUCKET(1, 2, 3, toUInt8(1))); +SELECT toTypeName(WIDTH_BUCKET(1, 2, 3, toUInt16(1))); +SELECT toTypeName(WIDTH_BUCKET(1, 2, 3, toUInt32(1))); +SELECT toTypeName(WIDTH_BUCKET(1, 2, 3, toUInt64(1))); +SELECT '----------'; +-- Test handling ColumnConst +SELECT WIDTH_BUCKET(1, low, high, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, 2, high, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(3, 3, high, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, low, 4, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(5, low, 5, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, 6, 6, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(7, 7, 7, count) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, low, high, 8) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(9, low, high, 9) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, 10, high, 10) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(11, 11, high, 11) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, low, 12, 12) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(13, low, 13, 13) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(operand, 14, 14, 14) FROM mytable WHERE count != 0; +SELECT WIDTH_BUCKET(15, 15, 15, 15) FROM mytable WHERE count != 0; \ No newline at end of file diff --git a/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference b/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference new file mode 100644 index 00000000000..73c1795f6f3 --- /dev/null +++ b/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference @@ -0,0 +1,37 @@ +-- { echoOn } + +SELECT a FROM ( select 1 AS a ) AS t1, ( select 2 AS b, 3 AS c) AS t2; +1 +SELECT a FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c) AS t2; +1 +1 +SELECT a FROM ( select 1 AS a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +1 +1 +SELECT a FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +1 +1 +1 +1 +SELECT a FROM ( select * from ( select 1 AS a UNION ALL select 1 as a) ) AS t1, ( select * from ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c )) AS t2; +1 +1 +1 +1 +SELECT b FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +2 +2 +2 +2 +SELECT c FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +3 +3 +3 +3 +SELECT 42 FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +42 +42 +42 +42 +SELECT count() FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +4 diff --git a/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql b/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql new file mode 100644 index 00000000000..10e5871cc44 --- /dev/null +++ b/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql @@ -0,0 +1,14 @@ +SET allow_experimental_analyzer = 1; +-- { echoOn } + +SELECT a FROM ( select 1 AS a ) AS t1, ( select 2 AS b, 3 AS c) AS t2; +SELECT a FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c) AS t2; +SELECT a FROM ( select 1 AS a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +SELECT a FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +SELECT a FROM ( select * from ( select 1 AS a UNION ALL select 1 as a) ) AS t1, ( select * from ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c )) AS t2; +SELECT b FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +SELECT c FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +SELECT 42 FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; +SELECT count() FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2; + + diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index ab03da8cc80..1ad7432a5bf 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -56,6 +56,7 @@ Hostname IPv IntN Integrations +invariants JSONAsString JSONAsObject JSONColumns @@ -130,6 +131,7 @@ ProtobufSingle QTCreator QueryCacheHits QueryCacheMisses +QEMU RBAC RawBLOB RedHat diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index d4abbdebee1..3814e94bf24 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -63,6 +63,7 @@ v22.4.5.9-stable 2022-05-06 v22.4.4.7-stable 2022-04-29 v22.4.3.3-stable 2022-04-26 v22.4.2.1-stable 2022-04-22 +v22.3.19.6-lts 2023-02-27 v22.3.18.37-lts 2023-02-15 v22.3.17.13-lts 2023-01-12 v22.3.16.1190-lts 2023-01-09