diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index b93c1b61ffd..e88a121b5d3 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -143,6 +143,8 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 + with: + fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync --recursive @@ -188,6 +190,8 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 + with: + fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync --recursive diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index cd516ba9674..66ba8547894 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -643,7 +643,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinTidy: + BuilderBinClangTidy: needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: @@ -1011,7 +1011,7 @@ jobs: - BuilderBinFreeBSD # - BuilderBinGCC - BuilderBinPPC64 - - BuilderBinTidy + - BuilderBinClangTidy - BuilderDebSplitted runs-on: [self-hosted, style-checker] steps: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 79d54d77f06..9cd8fd6f49d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -707,7 +707,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinTidy: + BuilderBinClangTidy: needs: [DockerHubPush, FastTest] runs-on: [self-hosted, builder] steps: @@ -1065,7 +1065,7 @@ jobs: - BuilderBinFreeBSD # - BuilderBinGCC - BuilderBinPPC64 - - BuilderBinTidy + - BuilderBinClangTidy - BuilderDebSplitted runs-on: [self-hosted, style-checker] if: ${{ success() || failure() }} diff --git a/.gitmodules b/.gitmodules index 55fd684fddb..c46b1c736fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -86,9 +86,6 @@ [submodule "contrib/h3"] path = contrib/h3 url = https://github.com/ClickHouse/h3 -[submodule "contrib/hyperscan"] - path = contrib/hyperscan - url = https://github.com/ClickHouse/hyperscan.git [submodule "contrib/libunwind"] path = contrib/libunwind url = https://github.com/ClickHouse/libunwind.git @@ -268,3 +265,12 @@ [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git +[submodule "contrib/vectorscan"] + path = contrib/vectorscan + url = https://github.com/VectorCamp/vectorscan.git +[submodule "contrib/liburing"] + path = contrib/liburing + url = https://github.com/axboe/liburing.git +[submodule "contrib/base-x"] + path = contrib/base-x + url = https://github.com/ClickHouse/base-x.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ea0ce2118..dfc51952250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,6 @@ * Add two new settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines` to allow skipping specified number of lines in the beginning of the file in CSV/TSV formats. [#37537](https://github.com/ClickHouse/ClickHouse/pull/37537) ([Kruglov Pavel](https://github.com/Avogar)). * `showCertificate` function shows current server's SSL certificate. [#37540](https://github.com/ClickHouse/ClickHouse/pull/37540) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * HTTP source for Data Dictionaries in Named Collections is supported. [#37581](https://github.com/ClickHouse/ClickHouse/pull/37581) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Added a new window function `nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL x SECOND])`. [#37628](https://github.com/ClickHouse/ClickHouse/pull/37628) ([Andrey Zvonov](https://github.com/zvonand)). * Implemented changing the comment for `ReplicatedMergeTree` tables. [#37416](https://github.com/ClickHouse/ClickHouse/pull/37416) ([Vasily Nemkov](https://github.com/Enmk)). * Added `SYSTEM UNFREEZE` query that deletes the whole backup regardless if the corresponding table is deleted or not. [#36424](https://github.com/ClickHouse/ClickHouse/pull/36424) ([Vadim Volodin](https://github.com/PolyProgrammist)). diff --git a/CMakeLists.txt b/CMakeLists.txt index c8bb1a2d1ca..4c7b732c68c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,10 +252,10 @@ else () endif () # Optionally split binaries and debug symbols. -option(INSTALL_STRIPPED_BINARIES "Split binaries and debug symbols" OFF) -if (INSTALL_STRIPPED_BINARIES) +option(SPLIT_DEBUG_SYMBOLS "Split binaries and debug symbols" OFF) +if (SPLIT_DEBUG_SYMBOLS) message(STATUS "Will split binaries and debug symbols") - set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information") + set(SPLITTED_DEBUG_SYMBOLS_DIR "stripped" CACHE STRING "A separate directory for stripped information") endif() cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd diff --git a/README.md b/README.md index e07a701d7c7..1d0146582a6 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,8 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. ## Upcoming events -* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/286304312/) Please join us for an evening of talks (in English), food and discussion. Featuring talks of ClickHouse in production and at least one on the deep internals of ClickHouse itself. +* [v22.7 Release Webinar](https://clickhouse.com/company/events/v22-7-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. +* [ClickHouse Meetup at the Cloudflare office in London](https://www.meetup.com/clickhouse-london-user-group/events/286891586/) ClickHouse meetup at the Cloudflare office space in central London +* [ClickHouse Meetup at the Metoda office in Munich](https://www.meetup.com/clickhouse-meetup-munich/events/286891667/) ClickHouse meetup at the Metoda office in Munich + + diff --git a/base/base/BorrowedObjectPool.h b/base/base/BorrowedObjectPool.h index 10dcc41d862..bb4c9cd1c21 100644 --- a/base/base/BorrowedObjectPool.h +++ b/base/base/BorrowedObjectPool.h @@ -89,7 +89,7 @@ public: inline void returnObject(T && object_to_return) { { - std::lock_guard lock(objects_mutex); + std::lock_guard lock(objects_mutex); objects.emplace_back(std::move(object_to_return)); --borrowed_objects_size; @@ -107,14 +107,14 @@ public: /// Allocated objects size by the pool. If allocatedObjectsSize == maxSize then pool is full. inline size_t allocatedObjectsSize() const { - std::unique_lock lock(objects_mutex); + std::lock_guard lock(objects_mutex); return allocated_objects_size; } /// Returns allocatedObjectsSize == maxSize inline bool isFull() const { - std::unique_lock lock(objects_mutex); + std::lock_guard lock(objects_mutex); return allocated_objects_size == max_size; } @@ -122,7 +122,7 @@ public: /// Then client will wait during borrowObject function call. inline size_t borrowedObjectsSize() const { - std::unique_lock lock(objects_mutex); + std::lock_guard lock(objects_mutex); return borrowed_objects_size; } diff --git a/base/base/argsToConfig.h b/base/base/argsToConfig.h index 134eed64fd2..9b7b44b7b7f 100644 --- a/base/base/argsToConfig.h +++ b/base/base/argsToConfig.h @@ -4,7 +4,7 @@ namespace Poco::Util { -class LayeredConfiguration; +class LayeredConfiguration; // NOLINT(cppcoreguidelines-virtual-class-destructor) } /// Import extra command line arguments to configuration. These are command line arguments after --. diff --git a/base/base/defines.h b/base/base/defines.h index 3959e690d71..a707e965675 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -124,21 +124,37 @@ #endif #endif -// Macros for Clang Thread Safety Analysis (TSA). They can be safely ignored by other compilers. -// Feel free to extend, but please stay close to https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#mutexheader +/// Macros for Clang Thread Safety Analysis (TSA). They can be safely ignored by other compilers. +/// Feel free to extend, but please stay close to https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#mutexheader #if defined(__clang__) -# define TSA_GUARDED_BY(...) __attribute__((guarded_by(__VA_ARGS__))) // data is protected by given capability -# define TSA_PT_GUARDED_BY(...) __attribute__((pt_guarded_by(__VA_ARGS__))) // pointed-to data is protected by the given capability -# define TSA_REQUIRES(...) __attribute__((requires_capability(__VA_ARGS__))) // thread needs exclusive possession of given capability -# define TSA_REQUIRES_SHARED(...) __attribute__((requires_shared_capability(__VA_ARGS__))) // thread needs shared possession of given capability -# define TSA_ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__))) // annotated lock must be locked after given lock -# define TSA_NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis)) // disable TSA for a function +# define TSA_GUARDED_BY(...) __attribute__((guarded_by(__VA_ARGS__))) /// data is protected by given capability +# define TSA_PT_GUARDED_BY(...) __attribute__((pt_guarded_by(__VA_ARGS__))) /// pointed-to data is protected by the given capability +# define TSA_REQUIRES(...) __attribute__((requires_capability(__VA_ARGS__))) /// thread needs exclusive possession of given capability +# define TSA_REQUIRES_SHARED(...) __attribute__((requires_shared_capability(__VA_ARGS__))) /// thread needs shared possession of given capability +# define TSA_ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__))) /// annotated lock must be locked after given lock +# define TSA_NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis)) /// disable TSA for a function + +/// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function) +/// Consider adding a comment before using these macros. +# define TSA_SUPPRESS_WARNING_FOR_READ(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> const auto & { return (x); }() +# define TSA_SUPPRESS_WARNING_FOR_WRITE(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> auto & { return (x); }() + +/// This macro is useful when only one thread writes to a member +/// and you want to read this member from the same thread without locking a mutex. +/// It's safe (because no concurrent writes are possible), but TSA generates a warning. +/// (Seems like there's no way to verify it, but it makes sense to distinguish it from TSA_SUPPRESS_WARNING_FOR_READ for readability) +# define TSA_READ_ONE_THREAD(x) TSA_SUPPRESS_WARNING_FOR_READ(x) + #else # define TSA_GUARDED_BY(...) # define TSA_PT_GUARDED_BY(...) # define TSA_REQUIRES(...) # define TSA_REQUIRES_SHARED(...) # define TSA_NO_THREAD_SAFETY_ANALYSIS + +# define TSA_SUPPRESS_WARNING_FOR_READ(x) +# define TSA_SUPPRESS_WARNING_FOR_WRITE(x) +# define TSA_READ_ONE_THREAD(x) #endif /// A template function for suppressing warnings about unused variables or function results. diff --git a/base/base/demangle.h b/base/base/demangle.h index 3c20920ab7c..ddca264ecab 100644 --- a/base/base/demangle.h +++ b/base/base/demangle.h @@ -27,6 +27,6 @@ struct FreeingDeleter } }; -typedef std::unique_ptr DemangleResult; +using DemangleResult = std::unique_ptr; DemangleResult tryDemangle(const char * name); diff --git a/base/base/strong_typedef.h b/base/base/strong_typedef.h index 0c2e9ca7e8e..c9ea30b73fd 100644 --- a/base/base/strong_typedef.h +++ b/base/base/strong_typedef.h @@ -23,10 +23,10 @@ public: constexpr StrongTypedef(): t() {} constexpr StrongTypedef(const Self &) = default; - constexpr StrongTypedef(Self &&) = default; + constexpr StrongTypedef(Self &&) noexcept(std::is_nothrow_move_constructible_v) = default; Self & operator=(const Self &) = default; - Self & operator=(Self &&) = default; + Self & operator=(Self &&) noexcept(std::is_nothrow_move_assignable_v)= default; template ::type> Self & operator=(const T & rhs) { t = rhs; return *this;} diff --git a/base/base/time.h b/base/base/time.h index d0b8e94a9a5..b029c9e3d42 100644 --- a/base/base/time.h +++ b/base/base/time.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #if defined (OS_DARWIN) || defined (OS_SUNOS) # define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC diff --git a/base/base/unaligned.h b/base/base/unaligned.h index ca73298adfb..013aa5274e9 100644 --- a/base/base/unaligned.h +++ b/base/base/unaligned.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/base/base/wide_integer.h b/base/base/wide_integer.h index de349633723..ffd30460c03 100644 --- a/base/base/wide_integer.h +++ b/base/base/wide_integer.h @@ -27,6 +27,8 @@ #include #include +// NOLINTBEGIN(*) + namespace wide { template @@ -257,4 +259,7 @@ struct hash>; } +// NOLINTEND(*) + #include "wide_integer_impl.h" + diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index e469e1683c8..5e32b286871 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -15,6 +15,8 @@ #include #include +// NOLINTBEGIN(*) + /// Use same extended double for all platforms #if (LDBL_MANT_DIG == 64) #define CONSTEXPR_FROM_DOUBLE constexpr @@ -1478,3 +1480,5 @@ struct hash> }; } + +// NOLINTEND(*) diff --git a/base/pcg-random/pcg_extras.hpp b/base/pcg-random/pcg_extras.hpp index 39c91c4ecfa..f5ba4d48849 100644 --- a/base/pcg-random/pcg_extras.hpp +++ b/base/pcg-random/pcg_extras.hpp @@ -90,6 +90,7 @@ #define PCG_EMULATED_128BIT_MATH 1 #endif +// NOLINTBEGIN(*) namespace pcg_extras { @@ -552,4 +553,6 @@ std::ostream& operator<<(std::ostream& out, printable_typename) { } // namespace pcg_extras +// NOLINTEND(*) + #endif // PCG_EXTRAS_HPP_INCLUDED diff --git a/base/pcg-random/pcg_random.hpp b/base/pcg-random/pcg_random.hpp index d9d3519a4cf..94e43e1007b 100644 --- a/base/pcg-random/pcg_random.hpp +++ b/base/pcg-random/pcg_random.hpp @@ -113,6 +113,8 @@ #include "pcg_extras.hpp" +// NOLINTBEGIN(*) + namespace DB { struct PcgSerializer; @@ -1777,4 +1779,6 @@ typedef pcg_engines::ext_oneseq_xsh_rs_64_32<14,32,true> pcg32_k16384_fast; #pragma warning(default:4146) #endif +// NOLINTEND(*) + #endif // PCG_RAND_HPP_INCLUDED diff --git a/base/widechar_width/widechar_width.h b/base/widechar_width/widechar_width.h index 3007a112886..4927b67eb91 100644 --- a/base/widechar_width/widechar_width.h +++ b/base/widechar_width/widechar_width.h @@ -16,6 +16,8 @@ #include #include +// NOLINTBEGIN(*) + /* Special width values */ enum { widechar_nonprint = -1, // The character is not printable. @@ -518,4 +520,6 @@ inline int widechar_wcwidth(wchar_t c) { return 1; } +// NOLINTEND(*) + #endif // WIDECHAR_WIDTH_H diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh deleted file mode 100755 index 5fea1bfbb88..00000000000 --- a/benchmark/benchmark.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env bash -# script to run query to databases - -function usage() -{ - cat < /proc/sys/vm/drop_caches" - - if [[ "$restart_server_each_query" == "1" && "$use_service" == "1" ]]; then - echo "restart server: $etc_init_d_service restart" - sudo $etc_init_d_service restart - fi - - for i in $(seq $TIMES) - do - if [[ -f $etc_init_d_service && "$use_service" == "1" ]]; then - sudo $etc_init_d_service status - server_status=$? - expect -f $expect_file "" - - if [[ "$?" != "0" || $server_status != "0" ]]; then - echo "restart server: $etc_init_d_service restart" - sudo $etc_init_d_service restart - fi - - #wait until can connect to server - restart_timer=0 - restart_limit=60 - expect -f $expect_file "" &> /dev/null - while [ "$?" != "0" ]; do - echo "waiting" - sleep 1 - let "restart_timer = $restart_timer + 1" - if (( $restart_limit < $restart_timer )); then - sudo $etc_init_d_service restart - restart_timer=0 - fi - expect -f $expect_file "" &> /dev/null - done - fi - - echo - echo "times: $i" - - echo "query:" "$query" - expect -f $expect_file "$query" - - done - fi - - let "index = $index + 1" - done -} - -temp_test_file=temp_queries_$table_name -cat $test_file | sed s/$table_name_pattern/$table_name/g > $temp_test_file -mapfile -t test_queries < $temp_test_file - -echo "start time: $(date)" -time execute "${test_queries[@]}" -echo "stop time: $(date)" diff --git a/benchmark/create_dump.sh b/benchmark/create_dump.sh deleted file mode 100755 index 3e26f8c1426..00000000000 --- a/benchmark/create_dump.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -table=hits_10m; time clickhouse-client --max_bytes_before_external_sort=30000000000 --query="SELECT toInt64(WatchID), JavaEnable, Title, GoodEvent, (EventTime < toDateTime('1971-01-01 00:00:00') ? toDateTime('1971-01-01 00:00:01') : EventTime), (EventDate < toDate('1971-01-01') ? toDate('1971-01-01') : EventDate), CounterID, ClientIP, RegionID, toInt64(UserID), CounterClass, OS, UserAgent, URL, Referer, Refresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, (ClientEventTime < toDateTime('1971-01-01 00:00:01') ? toDateTime('1971-01-01 00:00:01') : ClientEventTime), SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, toInt64(FUniqID), OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, (LocalEventTime < toDateTime('1971-01-01 00:00:01') ? toDateTime('1971-01-01 00:00:01') : LocalEventTime), Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, toInt64(RefererHash), toInt64(URLHash), CLID FROM $table ORDER BY rand()" | corrector_utf8 > /opt/dumps/${table}_corrected.tsv diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index f6206d0257c..e8c9c58aca3 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -1,107 +1,25 @@ #!/bin/bash -e -if [[ -n $1 ]]; then - SCALE=$1 -else - SCALE=100 -fi - -TABLE="hits_${SCALE}m_obfuscated" -DATASET="${TABLE}_v1.tar.xz" QUERIES_FILE="queries.sql" TRIES=3 -# Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'. +mkdir -p clickhouse-benchmark +pushd clickhouse-benchmark -FASTER_DOWNLOAD=wget -if command -v axel >/dev/null; then - FASTER_DOWNLOAD=axel -else - echo "It's recommended to install 'axel' for faster downloads." +# Download the binary +if [[ ! -x clickhouse ]]; then + curl https://clickhouse.com/ | sh fi -if command -v pixz >/dev/null; then - TAR_PARAMS='-Ipixz' -else - echo "It's recommended to install 'pixz' for faster decompression of the dataset." -fi - -mkdir -p clickhouse-benchmark-$SCALE -pushd clickhouse-benchmark-$SCALE - -OS=$(uname -s) -ARCH=$(uname -m) - -DIR= - -if [ "${OS}" = "Linux" ] -then - if [ "${ARCH}" = "x86_64" ] - then - DIR="amd64" - elif [ "${ARCH}" = "aarch64" ] - then - DIR="aarch64" - elif [ "${ARCH}" = "powerpc64le" ] - then - DIR="powerpc64le" - fi -elif [ "${OS}" = "FreeBSD" ] -then - if [ "${ARCH}" = "x86_64" ] - then - DIR="freebsd" - elif [ "${ARCH}" = "aarch64" ] - then - DIR="freebsd-aarch64" - elif [ "${ARCH}" = "powerpc64le" ] - then - DIR="freebsd-powerpc64le" - fi -elif [ "${OS}" = "Darwin" ] -then - if [ "${ARCH}" = "x86_64" ] - then - DIR="macos" - elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ] - then - DIR="macos-aarch64" - fi -fi - -if [ -z "${DIR}" ] -then - echo "The '${OS}' operating system with the '${ARCH}' architecture is not supported." - exit 1 -fi - -URL="https://builds.clickhouse.com/master/${DIR}/clickhouse" -echo -echo "Will download ${URL}" -echo -curl -O "${URL}" && chmod a+x clickhouse || exit 1 -echo -echo "Successfully downloaded the ClickHouse binary" - -chmod a+x clickhouse - if [[ ! -f $QUERIES_FILE ]]; then wget "https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/$QUERIES_FILE" fi -if [[ ! -d data ]]; then - if [[ ! -f $DATASET ]]; then - $FASTER_DOWNLOAD "https://datasets.clickhouse.com/hits/partitions/$DATASET" - fi - - tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET -fi - uptime echo "Starting clickhouse-server" -./clickhouse server > server.log 2>&1 & +./clickhouse server >/dev/null 2>&1 & PID=$! function finish { @@ -114,15 +32,36 @@ echo "Waiting for clickhouse-server to start" for i in {1..30}; do sleep 1 - ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM $TABLE" 2>/dev/null && break || echo '.' + ./clickhouse client --query "SELECT 'Ok.'" 2>/dev/null && break || echo -n '.' if [[ $i == 30 ]]; then exit 1; fi done +if [[ $(./clickhouse client --query "EXISTS hits") == '1' && $(./clickhouse client --query "SELECT count() FROM hits") == '100000000' ]]; then + echo "Dataset already downloaded" +else + echo "Will download the dataset" + ./clickhouse client --receive_timeout 1000 --max_insert_threads $(nproc || 4) --progress --query " + CREATE OR REPLACE TABLE hits ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) + AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" + + ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM hits" +fi + +if [[ $(./clickhouse client --query "SELECT count() FROM system.parts WHERE table = 'hits' AND database = 'default' AND active") == '1' ]]; then + echo "Dataset already prepared" +else + echo "Will prepare the dataset" + ./clickhouse client --receive_timeout 1000 --query "OPTIMIZE TABLE hits FINAL" +fi + echo echo "Will perform benchmark. Results:" echo -cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do +>result.csv +QUERY_NUM=1 + +cat "$QUERIES_FILE" | sed "s/{table}/hits/g" | while read query; do sync if [ "${OS}" = "Darwin" ] then @@ -133,11 +72,15 @@ cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do echo -n "[" for i in $(seq 1 $TRIES); do - RES=$(./clickhouse client --max_memory_usage 100G --time --format=Null --query="$query" 2>&1 ||:) + RES=$(./clickhouse client --time --format=Null --query="$query" 2>&1 ||:) [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" [[ "$i" != $TRIES ]] && echo -n ", " + + echo "${QUERY_NUM},${i},${RES}" >> result.csv done echo "]," + + QUERY_NUM=$((QUERY_NUM + 1)) done @@ -145,22 +88,23 @@ echo echo "Benchmark complete. System info:" echo +touch {cpu_model,cpu,df,memory,memory_total,blk,mdstat,instance}.txt + if [ "${OS}" = "Darwin" ] then echo '----Version, build id-----------' ./clickhouse local --query "SELECT format('Version: {}', version())" - sw_vers | grep BuildVersion ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" echo '----CPU-------------------------' - sysctl hw.model - sysctl -a | grep -E 'hw.activecpu|hw.memsize|hw.byteorder|cachesize' + sysctl hw.model | tee cpu_model.txt + sysctl -a | grep -E 'hw.activecpu|hw.memsize|hw.byteorder|cachesize' | tee cpu.txt echo '----Disk Free and Total--------' - df -h . + df -h . | tee df.txt echo '----Memory Free and Total-------' - vm_stat + vm_stat | tee memory.txt echo '----Physical Memory Amount------' - ls -l /var/vm + ls -l /var/vm | tee memory_total.txt echo '--------------------------------' else echo '----Version, build id-----------' @@ -168,22 +112,103 @@ else ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" echo '----CPU-------------------------' - cat /proc/cpuinfo | grep -i -F 'model name' | uniq - lscpu + cat /proc/cpuinfo | grep -i -F 'model name' | uniq | tee cpu_model.txt + lscpu | tee cpu.txt echo '----Block Devices---------------' - lsblk + lsblk | tee blk.txt echo '----Disk Free and Total--------' - df -h . + df -h . | tee df.txt echo '----Memory Free and Total-------' - free -h + free -h | tee memory.txt echo '----Physical Memory Amount------' - cat /proc/meminfo | grep MemTotal + cat /proc/meminfo | grep MemTotal | tee memory_total.txt echo '----RAID Info-------------------' - cat /proc/mdstat - #echo '----PCI-------------------------' - #lspci - #echo '----All Hardware Info-----------' - #lshw + cat /proc/mdstat| tee mdstat.txt echo '--------------------------------' fi echo + +echo "Instance type from IMDS (if available):" +curl -s --connect-timeout 1 'http://169.254.169.254/latest/meta-data/instance-type' | tee instance.txt +echo + +echo "Uploading the results (if possible)" + +UUID=$(./clickhouse local --query "SELECT generateUUIDv4()") + +./clickhouse local --query " + SELECT + '${UUID}' AS run_id, + version() AS version, + now() AS test_time, + (SELECT value FROM system.settings WHERE name = 'max_threads') AS threads, + filesystemCapacity() AS fs_capacity, + filesystemAvailable() AS fs_available, + file('cpu_model.txt') AS cpu_model, + file('cpu.txt') AS cpu, + file('df.txt') AS df, + file('memory.txt') AS memory, + file('memory_total.txt') AS memory_total, + file('blk.txt') AS blk, + file('mdstat.txt') AS mdstat, + file('instance.txt') AS instance +" | tee meta.tsv | ./clickhouse client --host play.clickhouse.com --secure --user benchmark --query " + INSERT INTO benchmark_runs + (run_id, version, test_time, threads, fs_capacity, fs_available, cpu_model, cpu, df, memory, memory_total, blk, mdstat, instance) + FORMAT TSV" || echo "Cannot upload results." + +./clickhouse local --query " + SELECT + '${UUID}' AS run_id, + c1 AS query_num, + c2 AS try_num, + c3 AS time + FROM file('result.csv') +" | tee results.tsv | ./clickhouse client --host play.clickhouse.com --secure --user benchmark --query " + INSERT INTO benchmark_results + (run_id, query_num, try_num, time) + FORMAT TSV" || echo "Cannot upload results. Please send the output to feedback@clickhouse.com" + +<" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/base-x b/contrib/base-x new file mode 160000 index 00000000000..a85f98fb4ed --- /dev/null +++ b/contrib/base-x @@ -0,0 +1 @@ +Subproject commit a85f98fb4ed52c2f4029a4b6ac1ef0bafdfc56f5 diff --git a/contrib/base-x-cmake/CMakeLists.txt b/contrib/base-x-cmake/CMakeLists.txt new file mode 100644 index 00000000000..b0c11a176c6 --- /dev/null +++ b/contrib/base-x-cmake/CMakeLists.txt @@ -0,0 +1,28 @@ +option (ENABLE_BASEX "Enable base-x" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_BASEX) + message(STATUS "Not using base-x") + return() +endif() + +set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base-x") + +set (SRCS + ${LIBRARY_DIR}/base_x.hh + ${LIBRARY_DIR}/uinteger_t.hh + ) + +add_library(_base-x INTERFACE) +target_include_directories(_base-x SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/base-x") + +if (XCODE OR XCODE_VERSION) + # https://gitlab.kitware.com/cmake/cmake/issues/17457 + # Some native build systems may not like targets that have only object files, so consider adding at least one real source file + # This applies to Xcode. + if (NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c" "") + endif () + target_sources(_base-x PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") +endif () + +add_library(ch_contrib::base-x ALIAS _base-x) \ No newline at end of file diff --git a/contrib/hyperscan b/contrib/hyperscan deleted file mode 160000 index 5edc68c5ac6..00000000000 --- a/contrib/hyperscan +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5edc68c5ac68d2d4f876159e9ee84def6d3dc87c diff --git a/contrib/libgsasl-cmake/CMakeLists.txt b/contrib/libgsasl-cmake/CMakeLists.txt index 3cf087c2f4c..f0c2395a831 100644 --- a/contrib/libgsasl-cmake/CMakeLists.txt +++ b/contrib/libgsasl-cmake/CMakeLists.txt @@ -113,4 +113,8 @@ if (TARGET ch_contrib::krb5) target_compile_definitions(_gsasl PRIVATE HAVE_GSSAPI_H=1 USE_GSSAPI=1) endif() +if (TARGET OpenSSL::SSL) + target_link_libraries(_gsasl PRIVATE OpenSSL::Crypto OpenSSL::SSL) +endif() + add_library(ch_contrib::gsasl ALIAS _gsasl) diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 5f4034a3a63..e39608998f5 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 5f4034a3a6376416504f17186c55fe401c6d8e5e +Subproject commit e39608998f5f6944ece9ec61f48e9172ec1de660 diff --git a/contrib/poco b/contrib/poco index de35b9fd72b..0e32cb42db7 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit de35b9fd72b57127abdc3a5beaf0e320d767e356 +Subproject commit 0e32cb42db76ddaa76848470219056908053b676 diff --git a/contrib/poco-cmake/XML/CMakeLists.txt b/contrib/poco-cmake/XML/CMakeLists.txt index 45100f11eb7..97e655a0f04 100644 --- a/contrib/poco-cmake/XML/CMakeLists.txt +++ b/contrib/poco-cmake/XML/CMakeLists.txt @@ -11,6 +11,7 @@ add_library (_poco_xml_expat ${SRCS_EXPAT}) add_library (Poco::XML::Expat ALIAS _poco_xml_expat) target_include_directories (_poco_xml_expat PUBLIC "${LIBRARY_DIR}/XML/include") +target_include_directories (_poco_xml_expat PRIVATE "${LIBRARY_DIR}/Foundation/include") # Poco::XML diff --git a/contrib/vectorscan b/contrib/vectorscan new file mode 160000 index 00000000000..73695e419c2 --- /dev/null +++ b/contrib/vectorscan @@ -0,0 +1 @@ +Subproject commit 73695e419c27af7fe2a099c7aa57931cc02aea5d diff --git a/contrib/hyperscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt similarity index 72% rename from contrib/hyperscan-cmake/CMakeLists.txt rename to contrib/vectorscan-cmake/CMakeLists.txt index 02c823a3a42..480ab3a384f 100644 --- a/contrib/hyperscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -1,54 +1,67 @@ -if (HAVE_SSSE3) - option (ENABLE_HYPERSCAN "Enable hyperscan library" ${ENABLE_LIBRARIES}) -elseif(ENABLE_HYPERSCAN) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use hyperscan without SSSE3") - set (ENABLE_HYPERSCAN OFF) -endif () +# We use vectorscan, a portable and API/ABI-compatible drop-in replacement for hyperscan. -if (NOT ENABLE_HYPERSCAN) - message (STATUS "Not using hyperscan") +if (ARCH_AMD64) + option (ENABLE_VECTORSCAN "Enable vectorscan library" ${ENABLE_LIBRARIES}) +endif() + +# TODO: vectorscan supports ARM yet some tests involving cyrillic letters fail (PR #38171) ... needs further investigation + +# TODO PPC should generally work but needs manual generation of ppc/config.h file on a PPC machine + +if (NOT ENABLE_VECTORSCAN) + message (STATUS "Not using vectorscan") return() endif() -set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/hyperscan") +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/vectorscan") +# Gobble up all c/cpp files in vectorscan/src/, omit *dump*.c/cpp files as we don't use the dump feature (see x86/config.h) set (SRCS "${LIBRARY_DIR}/src/alloc.c" + "${LIBRARY_DIR}/src/crc32.c" + "${LIBRARY_DIR}/src/database.c" + # "${LIBRARY_DIR}/src/dispatcher.c" # the linker's wrath be upon those who include dispatcher.c. + "${LIBRARY_DIR}/src/grey.cpp" + "${LIBRARY_DIR}/src/hs.cpp" + "${LIBRARY_DIR}/src/hs_valid_platform.c" + "${LIBRARY_DIR}/src/hs_version.c" + "${LIBRARY_DIR}/src/runtime.c" + "${LIBRARY_DIR}/src/scratch.c" + "${LIBRARY_DIR}/src/stream_compress.c" + "${LIBRARY_DIR}/src/compiler/asserts.cpp" "${LIBRARY_DIR}/src/compiler/compiler.cpp" "${LIBRARY_DIR}/src/compiler/error.cpp" - "${LIBRARY_DIR}/src/crc32.c" - "${LIBRARY_DIR}/src/database.c" + "${LIBRARY_DIR}/src/fdr/engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp" + "${LIBRARY_DIR}/src/fdr/fdr.c" "${LIBRARY_DIR}/src/fdr/fdr_compile.cpp" + "${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp" "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/fdr.c" "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" + "${LIBRARY_DIR}/src/fdr/teddy.c" + "${LIBRARY_DIR}/src/fdr/teddy_avx2.c" "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/teddy.c" - "${LIBRARY_DIR}/src/grey.cpp" - "${LIBRARY_DIR}/src/hs_valid_platform.c" - "${LIBRARY_DIR}/src/hs_version.c" - "${LIBRARY_DIR}/src/hs.cpp" + + "${LIBRARY_DIR}/src/hwlm/hwlm.c" "${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp" "${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp" - "${LIBRARY_DIR}/src/hwlm/hwlm.c" "${LIBRARY_DIR}/src/hwlm/noodle_build.cpp" - "${LIBRARY_DIR}/src/hwlm/noodle_engine.c" - "${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp" + "${LIBRARY_DIR}/src/hwlm/noodle_engine.cpp" + "${LIBRARY_DIR}/src/nfa/accel.c" + "${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp" "${LIBRARY_DIR}/src/nfa/accelcompile.cpp" "${LIBRARY_DIR}/src/nfa/castle.c" "${LIBRARY_DIR}/src/nfa/castlecompile.cpp" "${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp" "${LIBRARY_DIR}/src/nfa/dfa_min.cpp" "${LIBRARY_DIR}/src/nfa/gough.c" + "${LIBRARY_DIR}/src/nfa/goughcompile.cpp" "${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp" "${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp" - "${LIBRARY_DIR}/src/nfa/goughcompile.cpp" "${LIBRARY_DIR}/src/nfa/lbr.c" "${LIBRARY_DIR}/src/nfa/limex_64.c" "${LIBRARY_DIR}/src/nfa/limex_accel.c" @@ -59,28 +72,32 @@ set (SRCS "${LIBRARY_DIR}/src/nfa/limex_simd384.c" "${LIBRARY_DIR}/src/nfa/limex_simd512.c" "${LIBRARY_DIR}/src/nfa/mcclellan.c" - "${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp" "${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp" + "${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp" + "${LIBRARY_DIR}/src/nfa/mcsheng.c" "${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp" "${LIBRARY_DIR}/src/nfa/mcsheng_data.c" - "${LIBRARY_DIR}/src/nfa/mcsheng.c" "${LIBRARY_DIR}/src/nfa/mpv.c" "${LIBRARY_DIR}/src/nfa/mpvcompile.cpp" "${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c" "${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp" + "${LIBRARY_DIR}/src/nfa/rdfa.cpp" "${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp" "${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp" - "${LIBRARY_DIR}/src/nfa/rdfa.cpp" "${LIBRARY_DIR}/src/nfa/repeat.c" "${LIBRARY_DIR}/src/nfa/repeatcompile.cpp" "${LIBRARY_DIR}/src/nfa/sheng.c" "${LIBRARY_DIR}/src/nfa/shengcompile.cpp" - "${LIBRARY_DIR}/src/nfa/shufti.c" + "${LIBRARY_DIR}/src/nfa/shufti.cpp" "${LIBRARY_DIR}/src/nfa/shufticompile.cpp" "${LIBRARY_DIR}/src/nfa/tamarama.c" "${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp" - "${LIBRARY_DIR}/src/nfa/truffle.c" + "${LIBRARY_DIR}/src/nfa/truffle.cpp" "${LIBRARY_DIR}/src/nfa/trufflecompile.cpp" + "${LIBRARY_DIR}/src/nfa/vermicelli_simd.cpp" + "${LIBRARY_DIR}/src/nfa/vermicellicompile.cpp" + + "${LIBRARY_DIR}/src/nfagraph/ng.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp" @@ -100,8 +117,8 @@ set (SRCS "${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp" @@ -112,17 +129,17 @@ set (SRCS "${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_region.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_som.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_som.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_split.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp" @@ -132,10 +149,8 @@ set (SRCS "${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_width.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng.cpp" + "${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp" - "${LIBRARY_DIR}/src/parser/buildstate.cpp" - "${LIBRARY_DIR}/src/parser/check_refs.cpp" "${LIBRARY_DIR}/src/parser/Component.cpp" "${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp" "${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp" @@ -145,31 +160,34 @@ set (SRCS "${LIBRARY_DIR}/src/parser/ComponentByte.cpp" "${LIBRARY_DIR}/src/parser/ComponentClass.cpp" "${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp" - "${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp" "${LIBRARY_DIR}/src/parser/ComponentEUS.cpp" + "${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp" "${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp" "${LIBRARY_DIR}/src/parser/ComponentSequence.cpp" "${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp" "${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp" "${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp" - "${LIBRARY_DIR}/src/parser/control_verbs.cpp" + "${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp" + "${LIBRARY_DIR}/src/parser/buildstate.cpp" + "${LIBRARY_DIR}/src/parser/buildstate.cpp" + "${LIBRARY_DIR}/src/parser/check_refs.cpp" + "${LIBRARY_DIR}/src/parser/check_refs.cpp" "${LIBRARY_DIR}/src/parser/logical_combination.cpp" "${LIBRARY_DIR}/src/parser/parse_error.cpp" "${LIBRARY_DIR}/src/parser/parser_util.cpp" - "${LIBRARY_DIR}/src/parser/Parser.cpp" "${LIBRARY_DIR}/src/parser/prefilter.cpp" "${LIBRARY_DIR}/src/parser/shortcut_literal.cpp" "${LIBRARY_DIR}/src/parser/ucp_table.cpp" "${LIBRARY_DIR}/src/parser/unsupported.cpp" "${LIBRARY_DIR}/src/parser/utf8_validate.cpp" - "${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp" + "${LIBRARY_DIR}/src/rose/block.c" "${LIBRARY_DIR}/src/rose/catchup.c" "${LIBRARY_DIR}/src/rose/init.c" "${LIBRARY_DIR}/src/rose/match.c" "${LIBRARY_DIR}/src/rose/program_runtime.c" - "${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp" "${LIBRARY_DIR}/src/rose/rose_build_add.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp" "${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp" "${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp" "${LIBRARY_DIR}/src/rose/rose_build_castle.cpp" @@ -187,53 +205,95 @@ set (SRCS "${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp" "${LIBRARY_DIR}/src/rose/rose_build_merge.cpp" "${LIBRARY_DIR}/src/rose/rose_build_misc.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_misc.cpp" "${LIBRARY_DIR}/src/rose/rose_build_program.cpp" "${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp" "${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp" "${LIBRARY_DIR}/src/rose/rose_build_width.cpp" "${LIBRARY_DIR}/src/rose/rose_in_util.cpp" "${LIBRARY_DIR}/src/rose/stream.c" - "${LIBRARY_DIR}/src/runtime.c" - "${LIBRARY_DIR}/src/scratch.c" + "${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp" + "${LIBRARY_DIR}/src/som/slot_manager.cpp" "${LIBRARY_DIR}/src/som/som_runtime.c" "${LIBRARY_DIR}/src/som/som_stream.c" - "${LIBRARY_DIR}/src/stream_compress.c" + "${LIBRARY_DIR}/src/util/alloc.cpp" "${LIBRARY_DIR}/src/util/charreach.cpp" "${LIBRARY_DIR}/src/util/clique.cpp" "${LIBRARY_DIR}/src/util/compile_context.cpp" "${LIBRARY_DIR}/src/util/compile_error.cpp" - "${LIBRARY_DIR}/src/util/cpuid_flags.c" "${LIBRARY_DIR}/src/util/depth.cpp" "${LIBRARY_DIR}/src/util/fatbit_build.cpp" - "${LIBRARY_DIR}/src/util/multibit_build.cpp" "${LIBRARY_DIR}/src/util/multibit.c" + "${LIBRARY_DIR}/src/util/multibit_build.cpp" "${LIBRARY_DIR}/src/util/report_manager.cpp" - "${LIBRARY_DIR}/src/util/simd_utils.c" "${LIBRARY_DIR}/src/util/state_compress.c" "${LIBRARY_DIR}/src/util/target_info.cpp" "${LIBRARY_DIR}/src/util/ue2string.cpp" ) -add_library (_hyperscan ${SRCS}) +# The original build system invokes ragel on src/parser/{Parser|control_verbs}.rl (+ a few more .rl files which are unneeded). To avoid a +# build-time dependency on ragel (via contrib/ or find_program()), add the manually generated output of ragel to the sources. +# Please regenerate these files if you update vectorscan. +list (APPEND SRCS + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/Parser.cpp" + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/control_verbs.cpp" +) -target_compile_options (_hyperscan - PRIVATE -g0 # Library has too much debug information - -mno-avx -mno-avx2 # The library is using dynamic dispatch and is confused if AVX is enabled globally - -march=corei7 -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # The options from original build system - -fno-sanitize=undefined # Assume the library takes care of itself -) -target_include_directories (_hyperscan - PRIVATE - common - "${LIBRARY_DIR}/include" -) -target_include_directories (_hyperscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") +# Platform-dependent files if (ARCH_AMD64) - target_include_directories (_hyperscan PRIVATE x86_64) -endif () -target_link_libraries (_hyperscan PRIVATE boost::headers_only) + list(APPEND SRCS + "${LIBRARY_DIR}/src/util/arch/x86/cpuid_flags.c" + "${LIBRARY_DIR}/src/util/arch/x86/masked_move.c" + "${LIBRARY_DIR}/src/util/supervector/arch/x86/impl.cpp" + ) +endif() -add_library (ch_contrib::hyperscan ALIAS _hyperscan) +if (ARCH_AARCH64) + list(APPEND SRCS + "${LIBRARY_DIR}/src/util/arch/arm/cpuid_flags.c" + "${LIBRARY_DIR}/src/util/supervector/arch/arm/impl.cpp" + ) +endif() + +# TODO +# if (ARCH_PPC64LE) +# list(APPEND SRCS +# "${LIBRARY_DIR}/src/util/supervector/arch/ppc64el/impl.cpp" +# ) +# endif() + +add_library (_vectorscan ${SRCS}) + +target_compile_options (_vectorscan PRIVATE + -g0 # library has too much debug information + -fno-sanitize=undefined # assume the library takes care of itself + -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # options from original build system +) + +# Include version header manually generated by running the original build system +target_include_directories (_vectorscan SYSTEM PRIVATE common) + +# vectorscan inherited some patched in-source versions of boost headers to fix a bug in +# boost 1.69. This bug has been solved long ago but vectorscan's source code still +# points to the patched versions, so include it here. +target_include_directories (_vectorscan SYSTEM PRIVATE "${LIBRARY_DIR}/include") + +target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") + +# Include platform-specific config header generated by manually running the original build system +# Please regenerate these files if you update vectorscan. + +if (ARCH_AMD64) + target_include_directories (_vectorscan PRIVATE x86_64) +endif () + +if (ARCH_AARCH64) + target_include_directories (_vectorscan PRIVATE aarch64) +endif () + +target_link_libraries (_vectorscan PRIVATE boost::headers_only) + +add_library (ch_contrib::vectorscan ALIAS _vectorscan) diff --git a/contrib/vectorscan-cmake/aarch64/config.h b/contrib/vectorscan-cmake/aarch64/config.h new file mode 100644 index 00000000000..78da1c8ad00 --- /dev/null +++ b/contrib/vectorscan-cmake/aarch64/config.h @@ -0,0 +1,142 @@ +/* used by cmake */ + +#ifndef CONFIG_H_ +#define CONFIG_H_ + +/* "Define if the build is 32 bit" */ +/* #undef ARCH_32_BIT */ + +/* "Define if the build is 64 bit" */ +#define ARCH_64_BIT + +/* "Define if building for IA32" */ +/* #undef ARCH_IA32 */ + +/* "Define if building for EM64T" */ +/* #undef ARCH_X86_64 */ + +/* "Define if building for ARM32" */ +/* #undef ARCH_ARM32 */ + +/* "Define if building for AARCH64" */ +#define ARCH_AARCH64 + +/* "Define if building for PPC64EL" */ +/* #undef ARCH_PPC64EL */ + +/* "Define if cross compiling for AARCH64" */ +/* #undef CROSS_COMPILE_AARCH64 */ + +/* Define if building SVE for AARCH64. */ +/* #undef BUILD_SVE */ + +/* Define if building SVE2 for AARCH64. */ +/* #undef BUILD_SVE2 */ + +/* Define if building SVE2+BITPERM for AARCH64. */ +/* #undef BUILD_SVE2_BITPERM */ + +/* internal build, switch on dump support. */ +/* #undef DUMP_SUPPORT */ + +/* Define if building "fat" runtime. */ +/* #undef FAT_RUNTIME */ + +/* Define if building AVX2 in the fat runtime. */ +/* #undef BUILD_AVX2 */ + +/* Define if building AVX-512 in the fat runtime. */ +/* #undef BUILD_AVX512 */ + +/* Define if building AVX512VBMI in the fat runtime. */ +/* #undef BUILD_AVX512VBMI */ + +/* Define to 1 if `backtrace' works. */ +#define HAVE_BACKTRACE + +/* C compiler has __builtin_assume_aligned */ +#define HAVE_CC_BUILTIN_ASSUME_ALIGNED + +/* C++ compiler has __builtin_assume_aligned */ +#define HAVE_CXX_BUILTIN_ASSUME_ALIGNED + +/* C++ compiler has x86intrin.h */ +/* #undef HAVE_CXX_X86INTRIN_H */ + +/* C compiler has x86intrin.h */ +/* #undef HAVE_C_X86INTRIN_H */ + +/* C++ compiler has intrin.h */ +/* #undef HAVE_CXX_INTRIN_H */ + +/* C compiler has intrin.h */ +/* #undef HAVE_C_INTRIN_H */ + +/* C compiler has arm_neon.h */ +#define HAVE_C_ARM_NEON_H + +/* C compiler has arm_sve.h */ +/* #undef HAVE_C_ARM_SVE_H */ + +/* C compiler has arm_neon.h */ +/* #undef HAVE_C_PPC64EL_ALTIVEC_H */ + +/* Define to 1 if you have the declaration of `pthread_setaffinity_np', and to + 0 if you don't. */ +/* #undef HAVE_DECL_PTHREAD_SETAFFINITY_NP */ + +/* #undef HAVE_PTHREAD_NP_H */ + +/* Define to 1 if you have the `malloc_info' function. */ +/* #undef HAVE_MALLOC_INFO */ + +/* Define to 1 if you have the `memmem' function. */ +/* #undef HAVE_MEMMEM */ + +/* Define to 1 if you have a working `mmap' system call. */ +#define HAVE_MMAP + +/* Define to 1 if `posix_memalign' works. */ +#define HAVE_POSIX_MEMALIGN + +/* Define to 1 if you have the `setrlimit' function. */ +#define HAVE_SETRLIMIT + +/* Define to 1 if you have the `shmget' function. */ +/* #undef HAVE_SHMGET */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION + +/* Define to 1 if you have the `sigaltstack' function. */ +#define HAVE_SIGALTSTACK + +/* Define if the sqlite3_open_v2 call is available */ +/* #undef HAVE_SQLITE3_OPEN_V2 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the `_aligned_malloc' function. */ +/* #undef HAVE__ALIGNED_MALLOC */ + +/* Define if compiler has __builtin_constant_p */ +/* #undef HAVE__BUILTIN_CONSTANT_P */ + +/* Optimize, inline critical functions */ +#define HS_OPTIMIZE + +#define HS_VERSION +#define HS_MAJOR_VERSION +#define HS_MINOR_VERSION +#define HS_PATCH_VERSION + +#define BUILD_DATE + +/* define if this is a release build. */ +#define RELEASE_BUILD + +/* define if reverse_graph requires patch for boost 1.62.0 */ +/* #undef BOOST_REVGRAPH_PATCH */ + +#endif /* CONFIG_H_ */ diff --git a/contrib/hyperscan-cmake/common/hs_version.h b/contrib/vectorscan-cmake/common/hs_version.h similarity index 94% rename from contrib/hyperscan-cmake/common/hs_version.h rename to contrib/vectorscan-cmake/common/hs_version.h index f6fa8cb209f..8315b44fb2a 100644 --- a/contrib/hyperscan-cmake/common/hs_version.h +++ b/contrib/vectorscan-cmake/common/hs_version.h @@ -32,9 +32,8 @@ /** * A version string to identify this release of Hyperscan. */ -#define HS_VERSION_STRING "5.1.1 2000-01-01" +#define HS_VERSION_STRING "5.4.7 2022-06-20" -#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (1 << 8) | 0) +#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0) #endif /* HS_VERSION_H_C6428FAF8E3713 */ - diff --git a/contrib/vectorscan-cmake/rageled_files/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/Parser.cpp new file mode 100644 index 00000000000..aebbd7ace1e --- /dev/null +++ b/contrib/vectorscan-cmake/rageled_files/Parser.cpp @@ -0,0 +1,5605 @@ + +#line 1 "Parser.rl" +/* + * Copyright (c) 2015-2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Parser code (generated with Ragel from Parser.rl). + */ + +#include "config.h" + +/* Parser.cpp is a built source, may not be in same dir as parser files */ +#include "parser/check_refs.h" +#include "parser/control_verbs.h" +#include "parser/ComponentAlternation.h" +#include "parser/ComponentAssertion.h" +#include "parser/ComponentAtomicGroup.h" +#include "parser/ComponentBackReference.h" +#include "parser/ComponentBoundary.h" +#include "parser/ComponentByte.h" +#include "parser/ComponentClass.h" +#include "parser/ComponentCondReference.h" +#include "parser/ComponentEmpty.h" +#include "parser/ComponentEUS.h" +#include "parser/Component.h" +#include "parser/ComponentRepeat.h" +#include "parser/ComponentSequence.h" +#include "parser/ComponentWordBoundary.h" +#include "parser/parse_error.h" +#include "parser/Parser.h" +#include "ue2common.h" +#include "util/compare.h" +#include "util/flat_containers.h" +#include "util/unicode_def.h" +#include "util/verify_types.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +namespace ue2 { + +#define PUSH_SEQUENCE do {\ + sequences.push_back(ExprState(currentSeq, (size_t)(ts - ptr), \ + mode)); \ + } while(0) +#define POP_SEQUENCE do {\ + currentSeq = sequences.back().seq; \ + mode = sequences.back().mode; \ + sequences.pop_back(); \ + } while(0) + +namespace { + +/** \brief Structure representing current state as we're parsing (current + * sequence, current options). Stored in the 'sequences' vector. */ +struct ExprState { + ExprState(ComponentSequence *seq_in, size_t offset, + const ParseMode &mode_in) : + seq(seq_in), seqOffset(offset), mode(mode_in) {} + + ComponentSequence *seq; //!< current sequence + size_t seqOffset; //!< offset seq was entered, for error reporting + ParseMode mode; //!< current mode flags +}; + +} // namespace + +static +unsigned parseAsDecimal(unsigned oct) { + // The input was parsed as octal, but should have been parsed as decimal. + // Deconstruct the octal number and reconstruct into decimal + unsigned ret = 0; + unsigned multiplier = 1; + while (oct) { + ret += (oct & 0x7) * multiplier; + oct >>= 3; + multiplier *= 10; + } + return ret; +} + +/** \brief Maximum value for a positive integer. We use INT_MAX, as that's what + * PCRE uses. */ +static constexpr u32 MAX_NUMBER = INT_MAX; + +static +void pushDec(u32 *acc, char raw_digit) { + assert(raw_digit >= '0' && raw_digit <= '9'); + u32 digit_val = raw_digit - '0'; + + // Ensure that we don't overflow. + u64a val = ((u64a)*acc * 10) + digit_val; + if (val > MAX_NUMBER) { + throw LocatedParseError("Number is too big"); + } + + *acc = verify_u32(val); +} + +static +void pushOct(u32 *acc, char raw_digit) { + assert(raw_digit >= '0' && raw_digit <= '7'); + u32 digit_val = raw_digit - '0'; + + // Ensure that we don't overflow. + u64a val = ((u64a)*acc * 8) + digit_val; + if (val > MAX_NUMBER) { + throw LocatedParseError("Number is too big"); + } + + *acc = verify_u32(val); +} + +static +void throwInvalidRepeat(void) { + throw LocatedParseError("Invalid repeat"); +} + +static +void throwInvalidUtf8(void) { + throw ParseError("Expression is not valid UTF-8."); +} + +/** + * Adds the given child component to the parent sequence, returning a pointer + * to the new (child) "current sequence". + */ +static +ComponentSequence *enterSequence(ComponentSequence *parent, + unique_ptr child) { + assert(parent); + assert(child); + + ComponentSequence *seq = child.get(); + parent->addComponent(move(child)); + return seq; +} + +static +void addLiteral(ComponentSequence *currentSeq, char c, const ParseMode &mode) { + if (mode.utf8 && mode.caseless) { + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + assert(cc); + cc->add(c); + cc->finalize(); + currentSeq->addComponent(move(cc)); + } else { + currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); + } +} + +static +void addEscaped(ComponentSequence *currentSeq, unichar accum, + const ParseMode &mode, const char *err_msg) { + if (mode.utf8) { + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + assert(cc); + cc->add(accum); + cc->finalize(); + currentSeq->addComponent(move(cc)); + } else { + if (accum > 255) { + throw LocatedParseError(err_msg); + } + addLiteral(currentSeq, (char)accum, mode); + } +} + +static +void addEscapedOctal(ComponentSequence *currentSeq, unichar accum, + const ParseMode &mode) { + addEscaped(currentSeq, accum, mode, "Octal value is greater than \\377"); +} + +static +void addEscapedHex(ComponentSequence *currentSeq, unichar accum, + const ParseMode &mode) { + addEscaped(currentSeq, accum, mode, + "Hexadecimal value is greater than \\xFF"); +} + +#define SLASH_C_ERROR "\\c must be followed by an ASCII character" + +static +u8 decodeCtrl(char raw) { + if (raw & 0x80) { + throw LocatedParseError(SLASH_C_ERROR); + } + return mytoupper(raw) ^ 0x40; +} + +static +unichar readUtf8CodePoint2c(const char *s) { + auto *ts = (const u8 *)s; + assert(ts[0] >= 0xc0 && ts[0] < 0xe0); + assert(ts[1] >= 0x80 && ts[1] < 0xc0); + unichar val = ts[0] & 0x1f; + val <<= 6; + val |= ts[1] & 0x3f; + DEBUG_PRINTF("utf8 %02hhx %02hhx ->\\x{%x}\n", ts[0], + ts[1], val); + return val; +} + +static +unichar readUtf8CodePoint3c(const char *s) { + auto *ts = (const u8 *)s; + assert(ts[0] >= 0xe0 && ts[0] < 0xf0); + assert(ts[1] >= 0x80 && ts[1] < 0xc0); + assert(ts[2] >= 0x80 && ts[2] < 0xc0); + unichar val = ts[0] & 0x0f; + val <<= 6; + val |= ts[1] & 0x3f; + val <<= 6; + val |= ts[2] & 0x3f; + DEBUG_PRINTF("utf8 %02hhx %02hhx %02hhx ->\\x{%x}\n", ts[0], + ts[1], ts[2], val); + return val; +} + +static +unichar readUtf8CodePoint4c(const char *s) { + auto *ts = (const u8 *)s; + assert(ts[0] >= 0xf0 && ts[0] < 0xf8); + assert(ts[1] >= 0x80 && ts[1] < 0xc0); + assert(ts[2] >= 0x80 && ts[2] < 0xc0); + assert(ts[3] >= 0x80 && ts[3] < 0xc0); + unichar val = ts[0] & 0x07; + val <<= 6; + val |= ts[1] & 0x3f; + val <<= 6; + val |= ts[2] & 0x3f; + val <<= 6; + val |= ts[3] & 0x3f; + DEBUG_PRINTF("utf8 %02hhx %02hhx %02hhx %02hhx ->\\x{%x}\n", ts[0], + ts[1], ts[2], ts[3], val); + return val; +} + + +#line 1909 "Parser.rl" + + + +#line 281 "Parser.cpp" +static const short _regex_actions[] = { + 0, 1, 0, 1, 1, 1, 2, 1, + 3, 1, 4, 1, 7, 1, 8, 1, + 9, 1, 10, 1, 11, 1, 12, 1, + 13, 1, 15, 1, 16, 1, 17, 1, + 18, 1, 19, 1, 20, 1, 21, 1, + 22, 1, 23, 1, 24, 1, 25, 1, + 26, 1, 27, 1, 28, 1, 29, 1, + 30, 1, 31, 1, 32, 1, 33, 1, + 34, 1, 35, 1, 36, 1, 37, 1, + 38, 1, 39, 1, 40, 1, 41, 1, + 42, 1, 43, 1, 44, 1, 45, 1, + 46, 1, 47, 1, 48, 1, 49, 1, + 50, 1, 51, 1, 52, 1, 53, 1, + 54, 1, 55, 1, 56, 1, 57, 1, + 58, 1, 59, 1, 60, 1, 61, 1, + 62, 1, 63, 1, 64, 1, 65, 1, + 66, 1, 67, 1, 68, 1, 69, 1, + 70, 1, 71, 1, 72, 1, 73, 1, + 74, 1, 75, 1, 76, 1, 77, 1, + 78, 1, 79, 1, 80, 1, 81, 1, + 82, 1, 83, 1, 84, 1, 85, 1, + 86, 1, 87, 1, 88, 1, 89, 1, + 90, 1, 91, 1, 92, 1, 93, 1, + 94, 1, 95, 1, 96, 1, 97, 1, + 98, 1, 99, 1, 100, 1, 101, 1, + 102, 1, 103, 1, 104, 1, 105, 1, + 106, 1, 107, 1, 108, 1, 109, 1, + 110, 1, 111, 1, 112, 1, 113, 1, + 114, 1, 115, 1, 116, 1, 117, 1, + 118, 1, 119, 1, 120, 1, 121, 1, + 122, 1, 123, 1, 124, 1, 125, 1, + 126, 1, 127, 1, 128, 1, 129, 1, + 130, 1, 131, 1, 132, 1, 133, 1, + 134, 1, 135, 1, 136, 1, 137, 1, + 138, 1, 139, 1, 140, 1, 141, 1, + 142, 1, 143, 1, 144, 1, 145, 1, + 146, 1, 147, 1, 148, 1, 149, 1, + 150, 1, 151, 1, 152, 1, 153, 1, + 154, 1, 155, 1, 156, 1, 157, 1, + 158, 1, 159, 1, 160, 1, 161, 1, + 162, 1, 163, 1, 164, 1, 165, 1, + 166, 1, 167, 1, 168, 1, 169, 1, + 170, 1, 171, 1, 172, 1, 173, 1, + 174, 1, 175, 1, 176, 1, 177, 1, + 178, 1, 179, 1, 180, 1, 181, 1, + 182, 1, 183, 1, 184, 1, 185, 1, + 186, 1, 187, 1, 188, 1, 189, 1, + 190, 1, 191, 1, 192, 1, 193, 1, + 194, 1, 195, 1, 196, 1, 197, 1, + 198, 1, 199, 1, 200, 1, 201, 1, + 202, 1, 203, 1, 204, 1, 205, 1, + 206, 1, 207, 1, 208, 1, 209, 1, + 210, 1, 211, 1, 212, 1, 213, 1, + 214, 1, 215, 1, 216, 1, 217, 1, + 218, 1, 219, 1, 220, 1, 221, 1, + 222, 1, 223, 1, 224, 1, 225, 1, + 226, 1, 227, 1, 228, 1, 229, 1, + 230, 1, 231, 1, 232, 1, 233, 1, + 234, 1, 235, 1, 236, 1, 237, 1, + 240, 1, 242, 1, 243, 1, 244, 1, + 245, 1, 246, 1, 247, 1, 248, 1, + 249, 1, 250, 1, 251, 1, 252, 1, + 253, 1, 254, 1, 255, 1, 256, 1, + 257, 1, 258, 1, 259, 1, 260, 1, + 261, 1, 262, 1, 263, 1, 264, 1, + 265, 1, 266, 1, 267, 1, 268, 1, + 269, 1, 270, 1, 271, 1, 272, 1, + 273, 1, 274, 1, 275, 1, 276, 1, + 277, 1, 278, 1, 279, 1, 280, 1, + 281, 1, 282, 1, 283, 1, 284, 1, + 285, 1, 286, 1, 287, 1, 288, 1, + 289, 1, 290, 1, 291, 1, 292, 1, + 293, 1, 294, 1, 295, 1, 296, 1, + 297, 1, 298, 1, 299, 1, 300, 1, + 301, 1, 302, 1, 303, 1, 307, 1, + 308, 1, 309, 1, 310, 1, 311, 1, + 312, 1, 313, 1, 314, 1, 315, 1, + 316, 1, 317, 1, 318, 1, 319, 1, + 320, 1, 321, 1, 322, 1, 323, 1, + 324, 1, 325, 1, 326, 1, 327, 1, + 328, 1, 329, 1, 330, 1, 331, 1, + 332, 1, 333, 1, 334, 1, 335, 1, + 336, 1, 337, 1, 338, 1, 342, 1, + 343, 1, 344, 1, 345, 1, 346, 1, + 347, 1, 348, 1, 349, 1, 350, 1, + 352, 1, 353, 1, 354, 1, 355, 1, + 356, 1, 357, 1, 358, 1, 359, 1, + 360, 1, 361, 1, 362, 1, 363, 1, + 364, 1, 365, 1, 366, 1, 367, 1, + 368, 1, 369, 1, 370, 1, 371, 1, + 372, 1, 373, 1, 374, 1, 375, 1, + 376, 1, 377, 1, 378, 1, 379, 1, + 380, 1, 381, 1, 382, 1, 383, 1, + 384, 1, 385, 1, 386, 1, 387, 1, + 388, 1, 389, 1, 390, 1, 391, 1, + 392, 1, 393, 1, 394, 1, 395, 1, + 396, 1, 397, 1, 398, 1, 399, 1, + 400, 1, 401, 1, 402, 1, 403, 1, + 404, 1, 405, 1, 406, 1, 407, 1, + 408, 1, 409, 1, 410, 1, 411, 1, + 412, 1, 413, 1, 414, 1, 415, 1, + 416, 1, 417, 1, 418, 1, 419, 1, + 420, 1, 421, 1, 422, 1, 423, 1, + 424, 1, 425, 1, 426, 1, 427, 1, + 428, 1, 429, 1, 430, 1, 431, 1, + 432, 1, 433, 1, 434, 1, 435, 1, + 436, 2, 3, 0, 2, 4, 5, 2, + 5, 1, 2, 9, 10, 2, 9, 238, + 2, 9, 239, 2, 9, 339, 2, 10, + 1, 2, 10, 340, 2, 10, 341, 2, + 11, 241, 2, 11, 351, 2, 12, 241, + 2, 12, 351, 2, 13, 241, 2, 13, + 351, 2, 14, 375, 2, 14, 376, 2, + 25, 0, 2, 25, 3, 2, 25, 6, + 2, 25, 14, 3, 25, 5, 306, 3, + 25, 10, 305, 3, 25, 14, 15, 4, + 25, 9, 304, 10 +}; + +static const char _regex_cond_offsets[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 2, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 19, 20, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 27, 28, 29, 31, 31, 36, + 36, 37, 38, 39, 44, 44, 45, 46, + 47, 47 +}; + +static const char _regex_cond_lengths[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 6, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 2, 0, 5, 0, + 1, 1, 1, 5, 0, 1, 1, 1, + 0, 0 +}; + +static const short _regex_cond_keys[] = { + -128, -65, -128, -65, -128, -65, -128, -65, + -128, -65, -128, -65, -128, -65, -128, -65, + -128, -65, -128, -65, -128, -65, -128, -65, + -128, -65, -64, -33, -32, -17, -16, -9, + -8, -1, 35, 35, -128, -65, -128, -65, + -128, -65, -128, -65, -64, -33, -32, -17, + -16, -9, -8, -1, -128, -65, -128, -65, + -128, -65, 93, 93, 94, 94, -128, -65, + -64, -33, -32, -17, -16, -9, -8, -1, + -128, -65, -128, -65, -128, -65, -128, -65, + -64, -33, -32, -17, -16, -9, -8, -1, + -128, -65, -128, -65, -128, -65, 0 +}; + +static const char _regex_cond_spaces[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static const short _regex_key_offsets[] = { + 0, 0, 1, 23, 31, 39, 46, 54, + 55, 63, 71, 79, 86, 94, 97, 99, + 108, 115, 123, 131, 134, 140, 148, 151, + 158, 165, 173, 180, 184, 191, 194, 197, + 199, 202, 205, 207, 210, 213, 215, 216, + 218, 219, 227, 229, 232, 235, 236, 244, + 252, 260, 268, 275, 283, 290, 298, 305, + 313, 315, 318, 325, 329, 332, 335, 337, + 339, 341, 342, 344, 345, 347, 349, 350, + 351, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 369, + 370, 371, 372, 373, 374, 375, 376, 377, + 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 392, 393, 394, + 395, 396, 397, 399, 400, 401, 402, 403, + 404, 405, 406, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 429, + 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, 444, 445, + 446, 447, 448, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, + 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 516, 517, 519, 520, + 521, 522, 523, 524, 525, 526, 527, 528, + 529, 530, 531, 532, 533, 534, 535, 536, + 537, 538, 539, 540, 541, 542, 543, 544, + 545, 546, 547, 548, 549, 550, 551, 552, + 553, 554, 555, 556, 557, 558, 559, 561, + 562, 563, 564, 565, 566, 567, 568, 569, + 570, 571, 572, 573, 574, 575, 576, 577, + 578, 579, 580, 582, 583, 584, 585, 586, + 587, 588, 589, 590, 591, 592, 593, 594, + 595, 596, 597, 601, 602, 603, 604, 605, + 606, 607, 608, 609, 610, 611, 612, 613, + 614, 615, 616, 617, 618, 620, 621, 622, + 623, 624, 625, 626, 627, 628, 629, 631, + 632, 633, 634, 635, 636, 637, 640, 641, + 642, 643, 644, 645, 646, 647, 648, 650, + 651, 652, 653, 654, 655, 656, 658, 659, + 660, 661, 662, 663, 664, 665, 666, 667, + 668, 669, 670, 671, 672, 673, 674, 675, + 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 686, 687, 688, 689, 690, 691, + 692, 693, 694, 695, 696, 697, 698, 699, + 700, 701, 702, 704, 705, 706, 707, 708, + 709, 710, 714, 715, 716, 717, 718, 719, + 720, 721, 722, 723, 724, 725, 726, 727, + 728, 729, 730, 731, 732, 733, 734, 735, + 736, 737, 738, 739, 740, 741, 742, 743, + 744, 745, 746, 747, 748, 749, 750, 752, + 753, 754, 755, 756, 757, 758, 759, 760, + 761, 762, 763, 764, 765, 766, 767, 768, + 769, 770, 771, 773, 774, 775, 776, 777, + 778, 779, 780, 781, 782, 783, 784, 785, + 786, 787, 788, 789, 790, 791, 792, 793, + 794, 795, 796, 797, 798, 799, 800, 801, + 802, 803, 805, 806, 807, 808, 809, 810, + 811, 812, 813, 814, 815, 816, 817, 820, + 822, 823, 824, 825, 826, 827, 828, 829, + 830, 833, 834, 835, 836, 837, 838, 839, + 840, 841, 842, 843, 844, 845, 846, 847, + 849, 850, 851, 853, 854, 855, 856, 857, + 858, 859, 860, 861, 862, 863, 864, 865, + 866, 867, 868, 869, 870, 871, 872, 873, + 874, 875, 876, 877, 880, 883, 885, 900, + 903, 906, 908, 922, 927, 932, 936, 940, + 943, 946, 950, 954, 957, 960, 964, 968, + 972, 975, 978, 982, 986, 990, 994, 997, + 1000, 1004, 1008, 1012, 1016, 1019, 1022, 1026, + 1030, 1034, 1038, 1041, 1044, 1048, 1052, 1056, + 1060, 1063, 1066, 1070, 1074, 1078, 1082, 1085, + 1088, 1093, 1097, 1101, 1105, 1108, 1111, 1115, + 1119, 1123, 1126, 1129, 1133, 1137, 1141, 1145, + 1148, 1151, 1155, 1159, 1163, 1167, 1170, 1173, + 1177, 1181, 1185, 1188, 1191, 1195, 1199, 1203, + 1207, 1211, 1214, 1217, 1222, 1227, 1231, 1235, + 1238, 1241, 1245, 1249, 1252, 1255, 1259, 1263, + 1267, 1270, 1273, 1277, 1281, 1285, 1289, 1292, + 1295, 1299, 1303, 1307, 1311, 1314, 1317, 1321, + 1325, 1329, 1333, 1336, 1339, 1343, 1347, 1351, + 1355, 1358, 1361, 1365, 1369, 1373, 1377, 1380, + 1383, 1388, 1392, 1396, 1400, 1403, 1406, 1410, + 1414, 1418, 1421, 1424, 1428, 1432, 1436, 1440, + 1443, 1446, 1450, 1454, 1458, 1462, 1465, 1468, + 1472, 1476, 1480, 1483, 1486, 1490, 1494, 1498, + 1502, 1506, 1509, 1512, 1515, 1518, 1520, 1522, + 1525, 1532, 1534, 1536, 1538, 1540, 1542, 1544, + 1546, 1548, 1550, 1584, 1586, 1593, 1600, 1614, + 1616, 1622, 1625, 1634, 1635, 1638, 1641, 1648, + 1650, 1652, 1654, 1657, 1702, 1704, 1706, 1710, + 1714, 1716, 1717, 1717, 1723, 1725, 1727, 1729, + 1731, 1734, 1735, 1736, 1743, 1749, 1755, 1757, + 1759, 1761, 1763, 1765, 1767, 1768, 1771, 1794, + 1797, 1802, 1811, 1813, 1814, 1816, 1821, 1824, + 1826, 1828, 1829, 1831, 1841, 1847, 1848, 1853, + 1857, 1865, 1867, 1876, 1880, 1881, 1882, 1886, + 1887, 1890, 1890, 1897, 1913, 1916, 1955, 1957, + 1959, 1961, 1963, 1964, 1964, 1965, 1966, 1973, + 1979, 1985, 1987, 1989, 1991, 2000, 2002, 2015, + 2016, 2018, 2020, 2022, 2035, 2036, 2038, 2040, + 2042, 2043 +}; + +static const short _regex_trans_keys[] = { + 41, 33, 35, 38, 39, 40, 41, 43, + 45, 58, 60, 61, 62, 63, 67, 80, + 105, 109, 115, 120, 123, 48, 57, 41, + 95, 48, 57, 65, 90, 97, 122, 39, + 95, 48, 57, 65, 90, 97, 122, 95, + 48, 57, 65, 90, 97, 122, 39, 95, + 48, 57, 65, 90, 97, 122, 41, 41, + 95, 48, 57, 65, 90, 97, 122, 41, + 95, 48, 57, 65, 90, 97, 122, 41, + 95, 48, 57, 65, 90, 97, 122, 95, + 48, 57, 65, 90, 97, 122, 62, 95, + 48, 57, 65, 90, 97, 122, 33, 60, + 61, 33, 61, 38, 41, 95, 48, 57, + 65, 90, 97, 122, 95, 48, 57, 65, + 90, 97, 122, 41, 95, 48, 57, 65, + 90, 97, 122, 41, 95, 48, 57, 65, + 90, 97, 122, 41, 48, 57, 41, 58, + 105, 109, 115, 120, 62, 95, 48, 57, + 65, 90, 97, 122, 41, 48, 57, 95, + 48, 57, 65, 90, 97, 122, 95, 48, + 57, 65, 90, 97, 122, 41, 95, 48, + 57, 65, 90, 97, 122, 95, 48, 57, + 65, 90, 97, 122, 105, 109, 115, 120, + 41, 45, 58, 105, 109, 115, 120, 46, + 92, 93, 46, 92, 93, 46, 92, 58, + 92, 93, 58, 92, 93, 58, 92, 61, + 92, 93, 61, 92, 93, 61, 92, 39, + 48, 57, 62, 45, 95, 48, 57, 65, + 90, 97, 122, 48, 57, 125, 48, 57, + 125, 48, 57, 125, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 48, 57, 65, + 90, 97, 122, 39, 95, 48, 57, 65, + 90, 97, 122, 95, 48, 57, 65, 90, + 97, 122, 62, 95, 48, 57, 65, 90, + 97, 122, 95, 48, 57, 65, 90, 97, + 122, 95, 125, 48, 57, 65, 90, 97, + 122, 48, 55, 125, 48, 55, 125, 48, + 57, 65, 70, 97, 102, 44, 125, 48, + 57, 125, 48, 57, 125, 48, 57, 384, + 447, 384, 447, 384, 447, 41, 41, 80, + 41, 41, 70, 41, 56, 41, 121, 97, + 109, 98, 105, 99, 101, 110, 105, 97, + 110, 101, 115, 116, 97, 110, 108, 109, + 116, 105, 110, 101, 115, 101, 117, 109, + 97, 107, 110, 103, 97, 108, 105, 112, + 111, 109, 111, 102, 111, 97, 104, 105, + 109, 105, 108, 108, 101, 103, 104, 105, + 110, 101, 115, 101, 105, 100, 110, 114, + 97, 100, 105, 97, 110, 95, 65, 98, + 111, 114, 105, 103, 105, 110, 97, 108, + 105, 97, 110, 97, 101, 109, 114, 111, + 107, 101, 101, 109, 111, 110, 116, 105, + 99, 110, 101, 105, 102, 111, 114, 109, + 112, 114, 114, 105, 111, 116, 105, 108, + 108, 105, 99, 115, 118, 101, 114, 101, + 116, 97, 110, 97, 103, 97, 114, 105, + 121, 112, 116, 105, 97, 110, 95, 72, + 105, 101, 114, 111, 103, 108, 121, 112, + 104, 115, 104, 105, 111, 112, 105, 99, + 111, 114, 103, 105, 97, 110, 97, 103, + 111, 108, 105, 116, 105, 99, 116, 104, + 105, 99, 101, 101, 107, 106, 114, 97, + 114, 97, 116, 105, 109, 117, 107, 104, + 105, 110, 117, 108, 110, 111, 111, 98, + 114, 101, 119, 114, 97, 103, 97, 110, + 97, 112, 101, 114, 105, 97, 108, 95, + 65, 114, 97, 109, 97, 105, 99, 104, + 115, 101, 114, 105, 116, 101, 100, 99, + 114, 105, 112, 116, 105, 111, 110, 97, + 108, 95, 80, 97, 104, 114, 108, 97, + 118, 105, 116, 104, 105, 97, 110, 118, + 97, 110, 101, 115, 101, 105, 110, 116, + 121, 116, 104, 105, 110, 97, 100, 97, + 97, 107, 97, 110, 97, 97, 104, 95, + 76, 105, 97, 109, 114, 111, 115, 104, + 116, 104, 105, 101, 114, 111, 116, 105, + 110, 112, 99, 104, 97, 109, 110, 115, + 98, 117, 101, 97, 114, 95, 66, 117, + 99, 100, 105, 97, 110, 105, 97, 110, + 108, 110, 97, 121, 97, 108, 97, 109, + 100, 97, 105, 99, 116, 101, 105, 95, + 77, 97, 121, 101, 107, 110, 103, 111, + 108, 105, 97, 110, 97, 110, 109, 97, + 114, 119, 95, 84, 97, 105, 95, 76, + 117, 101, 111, 104, 97, 109, 95, 100, + 67, 104, 105, 107, 105, 95, 73, 80, + 83, 84, 116, 97, 108, 105, 99, 101, + 114, 115, 105, 97, 110, 111, 117, 116, + 104, 95, 65, 114, 97, 98, 105, 97, + 110, 117, 114, 107, 105, 99, 105, 121, + 97, 109, 97, 110, 121, 97, 97, 111, + 103, 115, 95, 80, 97, 101, 110, 105, + 99, 105, 97, 110, 106, 97, 110, 103, + 110, 105, 99, 109, 117, 97, 114, 105, + 116, 97, 110, 114, 97, 115, 104, 116, + 114, 97, 97, 118, 105, 97, 110, 110, + 104, 97, 108, 97, 110, 100, 97, 110, + 101, 115, 101, 108, 114, 111, 116, 105, + 95, 78, 97, 103, 114, 105, 105, 97, + 99, 103, 105, 109, 97, 98, 108, 111, + 103, 97, 110, 119, 97, 95, 76, 84, + 86, 101, 104, 97, 109, 105, 101, 116, + 105, 108, 108, 117, 103, 117, 97, 97, + 105, 110, 97, 98, 102, 101, 116, 97, + 110, 105, 110, 97, 103, 104, 97, 114, + 105, 116, 105, 99, 105, 110, 115, 112, + 100, 123, 94, 125, 94, 46, 92, 93, + 46, 92, 93, 46, 92, 58, 92, 93, + 94, 97, 98, 99, 100, 103, 108, 112, + 115, 117, 119, 120, 58, 92, 93, 58, + 92, 93, 58, 92, 58, 92, 93, 97, + 98, 99, 100, 103, 108, 112, 115, 117, + 119, 120, 58, 92, 93, 108, 115, 58, + 92, 93, 110, 112, 58, 92, 93, 117, + 58, 92, 93, 109, 58, 92, 93, 58, + 92, 93, 58, 92, 93, 104, 58, 92, + 93, 97, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 99, 58, 92, 93, 105, + 58, 92, 93, 105, 58, 92, 93, 58, + 92, 93, 58, 92, 93, 108, 58, 92, + 93, 97, 58, 92, 93, 110, 58, 92, + 93, 107, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 110, 58, 92, 93, 116, + 58, 92, 93, 114, 58, 92, 93, 108, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 105, 58, 92, 93, 103, 58, 92, + 93, 105, 58, 92, 93, 116, 58, 92, + 93, 58, 92, 93, 58, 92, 93, 114, + 58, 92, 93, 97, 58, 92, 93, 112, + 58, 92, 93, 104, 58, 92, 93, 58, + 92, 93, 58, 92, 93, 111, 58, 92, + 93, 119, 58, 92, 93, 101, 58, 92, + 93, 114, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 114, 117, 58, 92, 93, + 105, 58, 92, 93, 110, 58, 92, 93, + 116, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 110, 58, 92, 93, 99, 58, + 92, 93, 116, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 112, 58, 92, 93, + 97, 58, 92, 93, 99, 58, 92, 93, + 101, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 112, 58, 92, 93, 112, 58, + 92, 93, 101, 58, 92, 93, 114, 58, + 92, 93, 58, 92, 93, 58, 92, 93, + 111, 58, 92, 93, 114, 58, 92, 93, + 100, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 100, 58, 92, 93, 105, 58, + 92, 93, 103, 58, 92, 93, 105, 58, + 92, 93, 116, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 108, 115, 58, 92, + 93, 110, 112, 58, 92, 93, 117, 58, + 92, 93, 109, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 104, 58, 92, 93, + 97, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 99, 58, 92, 93, 105, 58, + 92, 93, 105, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 108, 58, 92, 93, + 97, 58, 92, 93, 110, 58, 92, 93, + 107, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 110, 58, 92, 93, 116, 58, + 92, 93, 114, 58, 92, 93, 108, 58, + 92, 93, 58, 92, 93, 58, 92, 93, + 105, 58, 92, 93, 103, 58, 92, 93, + 105, 58, 92, 93, 116, 58, 92, 93, + 58, 92, 93, 58, 92, 93, 114, 58, + 92, 93, 97, 58, 92, 93, 112, 58, + 92, 93, 104, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 111, 58, 92, 93, + 119, 58, 92, 93, 101, 58, 92, 93, + 114, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 114, 117, 58, 92, 93, 105, + 58, 92, 93, 110, 58, 92, 93, 116, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 110, 58, 92, 93, 99, 58, 92, + 93, 116, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 112, 58, 92, 93, 97, + 58, 92, 93, 99, 58, 92, 93, 101, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 112, 58, 92, 93, 112, 58, 92, + 93, 101, 58, 92, 93, 114, 58, 92, + 93, 58, 92, 93, 58, 92, 93, 111, + 58, 92, 93, 114, 58, 92, 93, 100, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 100, 58, 92, 93, 105, 58, 92, + 93, 103, 58, 92, 93, 105, 58, 92, + 93, 116, 58, 92, 93, 58, 92, 93, + 61, 92, 93, 61, 92, 93, 61, 92, + 48, 55, 125, 48, 55, 125, 48, 57, + 65, 70, 97, 102, 384, 447, 384, 447, + 384, 447, 384, 447, 384, 447, 384, 447, + 384, 447, 384, 447, 384, 447, 0, 32, + 36, 40, 41, 42, 43, 46, 63, 91, + 92, 94, 123, 124, 1315, 1571, 1, 8, + 9, 13, 14, 34, 37, 255, 384, 447, + 448, 479, 480, 495, 496, 503, 504, 511, + 42, 63, 95, 48, 57, 65, 90, 97, + 122, 95, 48, 57, 65, 90, 97, 122, + 39, 48, 60, 63, 82, 95, 49, 55, + 56, 57, 65, 90, 97, 122, 48, 57, + 105, 109, 115, 120, 48, 57, 41, 48, + 57, 33, 61, 95, 48, 57, 65, 90, + 97, 122, 123, 41, 48, 57, 60, 61, + 62, 41, 45, 58, 105, 109, 115, 120, + 43, 63, 43, 63, 43, 63, 46, 58, + 61, 48, 65, 66, 67, 68, 69, 71, + 72, 75, 76, 78, 80, 81, 82, 83, + 85, 86, 87, 88, 90, 97, 98, 99, + 100, 101, 102, 103, 104, 107, 108, 110, + 111, 112, 114, 115, 116, 117, 118, 119, + 120, 122, 49, 55, 56, 57, 48, 55, + 48, 55, 48, 55, 56, 57, 48, 55, + 56, 57, 48, 57, 123, 39, 45, 60, + 123, 48, 57, 48, 57, 48, 57, 48, + 57, 48, 57, 39, 60, 123, 123, 123, + 123, 48, 57, 65, 70, 97, 102, 48, + 57, 65, 70, 97, 102, 48, 57, 65, + 70, 97, 102, 48, 57, 43, 63, 384, + 447, 384, 447, 384, 447, 41, 85, 41, + 41, 67, 84, 65, 66, 67, 68, 69, + 71, 72, 73, 74, 75, 76, 77, 78, + 79, 80, 82, 83, 84, 85, 86, 88, + 89, 90, 110, 114, 118, 97, 101, 111, + 114, 117, 97, 99, 102, 104, 110, 111, + 115, 117, 121, 109, 112, 101, 103, 116, + 101, 108, 111, 114, 117, 97, 101, 105, + 103, 117, 109, 110, 97, 97, 104, 38, + 97, 101, 105, 108, 109, 111, 116, 117, + 121, 97, 99, 101, 110, 111, 121, 101, + 100, 101, 107, 108, 111, 103, 108, 114, + 115, 99, 100, 101, 102, 104, 105, 111, + 115, 101, 117, 97, 99, 104, 105, 107, + 109, 111, 117, 121, 97, 101, 104, 105, + 103, 97, 97, 112, 115, 119, 105, 108, + 112, 115, 67, 76, 77, 78, 80, 83, + 90, 45, 91, 92, 93, 0, 255, 384, + 447, 448, 479, 480, 495, 496, 503, 504, + 511, 46, 58, 61, 48, 68, 69, 72, + 76, 78, 80, 81, 83, 85, 86, 87, + 97, 98, 99, 100, 101, 102, 103, 104, + 108, 110, 111, 112, 114, 115, 116, 117, + 118, 119, 120, 49, 55, 56, 57, 65, + 90, 105, 122, 48, 55, 48, 55, 48, + 55, 48, 55, 123, 123, 123, 123, 48, + 57, 65, 70, 97, 102, 48, 57, 65, + 70, 97, 102, 48, 57, 65, 70, 97, + 102, 384, 447, 384, 447, 384, 447, 92, + 1117, 1118, -128, 91, 95, 127, 861, 862, + 69, 81, 92, 0, 255, 384, 447, 448, + 479, 480, 495, 496, 503, 504, 511, 69, + 384, 447, 384, 447, 384, 447, 92, 0, + 255, 384, 447, 448, 479, 480, 495, 496, + 503, 504, 511, 69, 384, 447, 384, 447, + 384, 447, 41, 10, 0 +}; + +static const char _regex_single_lengths[] = { + 0, 1, 20, 2, 2, 1, 2, 1, + 2, 2, 2, 1, 2, 3, 2, 3, + 1, 2, 2, 1, 6, 2, 1, 1, + 1, 2, 1, 4, 7, 3, 3, 2, + 3, 3, 2, 3, 3, 2, 1, 0, + 1, 2, 0, 1, 1, 1, 2, 2, + 2, 2, 1, 2, 1, 2, 1, 2, + 0, 1, 1, 2, 1, 1, 0, 0, + 0, 1, 2, 1, 2, 2, 1, 1, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 4, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 3, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 4, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 2, + 1, 1, 1, 1, 1, 1, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 2, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 3, 3, 2, 15, 3, + 3, 2, 14, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, + 4, 3, 3, 4, 4, 4, 4, 3, + 3, 4, 4, 4, 4, 3, 3, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 4, 3, 3, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, + 4, 3, 3, 4, 4, 4, 4, 3, + 3, 4, 4, 4, 4, 3, 3, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 4, 3, 3, 3, 3, 2, 0, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 16, 2, 1, 1, 6, 0, + 4, 1, 3, 1, 1, 3, 7, 2, + 2, 2, 3, 41, 0, 0, 0, 0, + 0, 1, 0, 4, 0, 0, 0, 0, + 3, 1, 1, 1, 0, 0, 0, 2, + 0, 0, 0, 2, 1, 3, 23, 3, + 5, 9, 2, 1, 2, 5, 3, 2, + 2, 1, 2, 10, 6, 1, 5, 4, + 8, 2, 9, 4, 1, 1, 4, 1, + 3, 0, 7, 4, 3, 31, 0, 0, + 0, 0, 1, 0, 1, 1, 1, 0, + 0, 0, 0, 0, 3, 2, 1, 1, + 0, 0, 0, 1, 1, 0, 0, 0, + 1, 1 +}; + +static const char _regex_range_lengths[] = { + 0, 0, 1, 3, 3, 3, 3, 0, + 3, 3, 3, 3, 3, 0, 0, 3, + 3, 3, 3, 1, 0, 3, 1, 3, + 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 3, 1, 1, 1, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 3, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 9, 0, 3, 3, 4, 1, + 1, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 2, 1, 1, 2, 2, + 1, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 1, 0, + 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 6, 0, 4, 1, 1, + 1, 1, 0, 0, 0, 0, 3, 3, + 3, 1, 1, 1, 3, 0, 6, 0, + 1, 1, 1, 6, 0, 1, 1, 1, + 0, 0 +}; + +static const short _regex_index_offsets[] = { + 0, 0, 2, 24, 30, 36, 41, 47, + 49, 55, 61, 67, 72, 78, 82, 85, + 92, 97, 103, 109, 112, 119, 125, 128, + 133, 138, 144, 149, 154, 162, 166, 170, + 173, 177, 181, 184, 188, 192, 195, 197, + 199, 201, 207, 209, 212, 215, 217, 223, + 229, 235, 241, 246, 252, 257, 263, 268, + 274, 276, 279, 284, 288, 291, 294, 296, + 298, 300, 302, 305, 307, 310, 313, 315, + 317, 320, 322, 324, 326, 328, 330, 332, + 334, 336, 338, 340, 342, 344, 346, 350, + 352, 354, 356, 358, 360, 362, 364, 366, + 368, 370, 372, 374, 376, 378, 380, 382, + 384, 386, 388, 390, 392, 395, 397, 399, + 401, 403, 405, 408, 410, 412, 414, 416, + 418, 420, 422, 425, 427, 429, 431, 433, + 435, 437, 439, 441, 443, 445, 447, 449, + 451, 453, 455, 457, 459, 461, 463, 466, + 468, 470, 472, 474, 476, 478, 480, 482, + 484, 486, 488, 490, 492, 494, 496, 498, + 500, 502, 504, 507, 509, 511, 513, 515, + 517, 519, 521, 523, 525, 528, 530, 532, + 534, 536, 538, 540, 542, 544, 546, 548, + 550, 552, 554, 556, 558, 560, 562, 564, + 566, 568, 570, 572, 574, 576, 578, 580, + 582, 584, 586, 588, 590, 592, 594, 596, + 598, 600, 602, 604, 606, 608, 610, 612, + 614, 616, 618, 620, 622, 624, 626, 628, + 630, 632, 634, 636, 638, 640, 643, 645, + 647, 649, 651, 653, 655, 657, 659, 661, + 663, 665, 667, 669, 671, 673, 675, 677, + 679, 681, 683, 685, 687, 689, 691, 693, + 695, 697, 699, 701, 703, 705, 707, 709, + 711, 713, 715, 717, 719, 721, 723, 726, + 728, 730, 732, 734, 736, 738, 740, 742, + 744, 746, 748, 750, 752, 754, 756, 758, + 760, 762, 764, 767, 769, 771, 773, 775, + 777, 779, 781, 783, 785, 787, 789, 791, + 793, 795, 797, 802, 804, 806, 808, 810, + 812, 814, 816, 818, 820, 822, 824, 826, + 828, 830, 832, 834, 836, 839, 841, 843, + 845, 847, 849, 851, 853, 855, 857, 860, + 862, 864, 866, 868, 870, 872, 876, 878, + 880, 882, 884, 886, 888, 890, 892, 895, + 897, 899, 901, 903, 905, 907, 910, 912, + 914, 916, 918, 920, 922, 924, 926, 928, + 930, 932, 934, 936, 938, 940, 942, 944, + 946, 948, 950, 952, 954, 956, 958, 960, + 962, 964, 966, 968, 970, 972, 974, 976, + 978, 980, 982, 984, 986, 988, 990, 992, + 994, 996, 998, 1001, 1003, 1005, 1007, 1009, + 1011, 1013, 1018, 1020, 1022, 1024, 1026, 1028, + 1030, 1032, 1034, 1036, 1038, 1040, 1042, 1044, + 1046, 1048, 1050, 1052, 1054, 1056, 1058, 1060, + 1062, 1064, 1066, 1068, 1070, 1072, 1074, 1076, + 1078, 1080, 1082, 1084, 1086, 1088, 1090, 1093, + 1095, 1097, 1099, 1101, 1103, 1105, 1107, 1109, + 1111, 1113, 1115, 1117, 1119, 1121, 1123, 1125, + 1127, 1129, 1131, 1134, 1136, 1138, 1140, 1142, + 1144, 1146, 1148, 1150, 1152, 1154, 1156, 1158, + 1160, 1162, 1164, 1166, 1168, 1170, 1172, 1174, + 1176, 1178, 1180, 1182, 1184, 1186, 1188, 1190, + 1192, 1194, 1197, 1199, 1201, 1203, 1205, 1207, + 1209, 1211, 1213, 1215, 1217, 1219, 1221, 1225, + 1228, 1230, 1232, 1234, 1236, 1238, 1240, 1242, + 1244, 1248, 1250, 1252, 1254, 1256, 1258, 1260, + 1262, 1264, 1266, 1268, 1270, 1272, 1274, 1276, + 1279, 1281, 1283, 1286, 1288, 1290, 1292, 1294, + 1296, 1298, 1300, 1302, 1304, 1306, 1308, 1310, + 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, + 1328, 1330, 1332, 1334, 1338, 1342, 1345, 1361, + 1365, 1369, 1372, 1387, 1393, 1399, 1404, 1409, + 1413, 1417, 1422, 1427, 1431, 1435, 1440, 1445, + 1450, 1454, 1458, 1463, 1468, 1473, 1478, 1482, + 1486, 1491, 1496, 1501, 1506, 1510, 1514, 1519, + 1524, 1529, 1534, 1538, 1542, 1547, 1552, 1557, + 1562, 1566, 1570, 1575, 1580, 1585, 1590, 1594, + 1598, 1604, 1609, 1614, 1619, 1623, 1627, 1632, + 1637, 1642, 1646, 1650, 1655, 1660, 1665, 1670, + 1674, 1678, 1683, 1688, 1693, 1698, 1702, 1706, + 1711, 1716, 1721, 1725, 1729, 1734, 1739, 1744, + 1749, 1754, 1758, 1762, 1768, 1774, 1779, 1784, + 1788, 1792, 1797, 1802, 1806, 1810, 1815, 1820, + 1825, 1829, 1833, 1838, 1843, 1848, 1853, 1857, + 1861, 1866, 1871, 1876, 1881, 1885, 1889, 1894, + 1899, 1904, 1909, 1913, 1917, 1922, 1927, 1932, + 1937, 1941, 1945, 1950, 1955, 1960, 1965, 1969, + 1973, 1979, 1984, 1989, 1994, 1998, 2002, 2007, + 2012, 2017, 2021, 2025, 2030, 2035, 2040, 2045, + 2049, 2053, 2058, 2063, 2068, 2073, 2077, 2081, + 2086, 2091, 2096, 2100, 2104, 2109, 2114, 2119, + 2124, 2129, 2133, 2137, 2141, 2145, 2148, 2150, + 2153, 2158, 2160, 2162, 2164, 2166, 2168, 2170, + 2172, 2174, 2176, 2202, 2205, 2210, 2215, 2226, + 2228, 2234, 2237, 2244, 2246, 2249, 2253, 2261, + 2264, 2267, 2270, 2274, 2318, 2320, 2322, 2325, + 2328, 2330, 2332, 2333, 2339, 2341, 2343, 2345, + 2347, 2351, 2353, 2355, 2360, 2364, 2368, 2370, + 2373, 2375, 2377, 2379, 2382, 2384, 2388, 2412, + 2416, 2422, 2432, 2435, 2437, 2440, 2446, 2450, + 2453, 2456, 2458, 2461, 2472, 2479, 2481, 2487, + 2492, 2501, 2504, 2514, 2519, 2521, 2523, 2528, + 2530, 2534, 2535, 2543, 2554, 2558, 2594, 2596, + 2598, 2600, 2602, 2604, 2605, 2607, 2609, 2614, + 2618, 2622, 2624, 2626, 2628, 2635, 2638, 2646, + 2648, 2650, 2652, 2654, 2662, 2664, 2666, 2668, + 2670, 2672 +}; + +static const short _regex_indicies[] = { + 0, 1, 3, 4, 5, 6, 7, 8, + 9, 10, 12, 13, 14, 15, 16, 17, + 18, 19, 19, 19, 19, 20, 11, 2, + 22, 23, 23, 23, 23, 21, 24, 25, + 25, 25, 25, 21, 27, 27, 27, 27, + 26, 28, 27, 27, 27, 27, 26, 29, + 26, 29, 30, 30, 30, 30, 26, 31, + 30, 32, 30, 30, 26, 29, 30, 32, + 30, 30, 26, 33, 33, 33, 33, 26, + 28, 33, 33, 33, 33, 26, 34, 35, + 36, 26, 37, 38, 26, 39, 40, 30, + 41, 30, 30, 26, 42, 42, 42, 42, + 26, 40, 42, 42, 42, 42, 26, 40, + 30, 41, 30, 30, 26, 43, 44, 21, + 45, 46, 47, 47, 47, 47, 21, 24, + 48, 48, 48, 48, 21, 49, 50, 21, + 48, 48, 48, 48, 21, 51, 51, 51, + 51, 21, 52, 51, 51, 51, 51, 21, + 23, 23, 23, 23, 21, 47, 47, 47, + 47, 21, 45, 53, 46, 54, 54, 54, + 54, 21, 57, 58, 55, 56, 57, 58, + 59, 56, 57, 58, 56, 61, 62, 55, + 60, 61, 62, 63, 60, 61, 62, 60, + 65, 66, 55, 64, 65, 66, 59, 64, + 65, 66, 64, 69, 68, 70, 67, 69, + 71, 72, 74, 73, 74, 74, 67, 75, + 67, 77, 76, 67, 77, 78, 67, 77, + 67, 74, 80, 79, 74, 74, 67, 74, + 80, 81, 74, 74, 67, 74, 80, 74, + 74, 74, 67, 74, 82, 74, 74, 74, + 67, 84, 84, 84, 84, 83, 85, 84, + 84, 84, 84, 83, 86, 86, 86, 86, + 83, 87, 86, 86, 86, 86, 83, 88, + 88, 88, 88, 83, 88, 89, 88, 88, + 88, 83, 91, 90, 92, 91, 90, 95, + 94, 94, 94, 93, 97, 99, 98, 96, + 101, 100, 96, 102, 100, 96, 104, 103, + 105, 103, 106, 103, 109, 108, 109, 110, + 108, 111, 108, 109, 112, 108, 113, 114, + 108, 115, 108, 117, 116, 118, 119, 116, + 120, 116, 121, 116, 122, 116, 123, 116, + 124, 116, 125, 116, 126, 116, 127, 116, + 128, 116, 129, 116, 130, 116, 131, 116, + 132, 116, 133, 134, 135, 116, 136, 116, + 137, 116, 138, 116, 139, 116, 140, 116, + 141, 116, 142, 116, 143, 116, 144, 116, + 145, 116, 146, 116, 147, 116, 148, 116, + 149, 116, 150, 116, 151, 116, 152, 116, + 153, 116, 154, 116, 155, 116, 156, 116, + 157, 158, 116, 159, 116, 160, 116, 161, + 116, 162, 116, 163, 116, 164, 165, 116, + 166, 116, 167, 116, 168, 116, 169, 116, + 170, 116, 171, 116, 172, 116, 174, 175, + 173, 176, 173, 177, 173, 178, 173, 179, + 173, 180, 173, 181, 173, 182, 173, 183, + 173, 184, 173, 185, 173, 186, 173, 187, + 173, 188, 173, 189, 173, 190, 173, 191, + 173, 192, 173, 193, 173, 194, 173, 195, + 196, 173, 197, 173, 198, 173, 199, 173, + 200, 173, 201, 173, 202, 173, 204, 203, + 205, 203, 206, 203, 207, 203, 208, 203, + 209, 203, 210, 173, 211, 173, 212, 173, + 213, 173, 214, 173, 215, 173, 216, 173, + 217, 218, 173, 219, 173, 220, 173, 221, + 173, 222, 173, 223, 173, 224, 173, 225, + 173, 226, 173, 227, 173, 228, 229, 116, + 230, 116, 231, 116, 232, 116, 233, 116, + 234, 116, 235, 116, 236, 116, 237, 116, + 238, 116, 239, 116, 240, 116, 241, 116, + 242, 116, 243, 116, 244, 116, 245, 116, + 246, 116, 247, 116, 248, 116, 249, 116, + 250, 116, 251, 116, 252, 116, 253, 116, + 254, 116, 255, 116, 256, 116, 257, 116, + 258, 116, 259, 116, 260, 116, 261, 116, + 262, 116, 263, 116, 264, 116, 265, 116, + 266, 116, 267, 116, 268, 116, 269, 116, + 270, 116, 271, 116, 272, 116, 273, 116, + 274, 116, 275, 116, 276, 116, 277, 116, + 278, 116, 279, 116, 280, 116, 281, 116, + 282, 116, 283, 116, 284, 116, 285, 116, + 286, 287, 116, 288, 116, 289, 116, 290, + 116, 291, 116, 292, 116, 293, 116, 294, + 116, 295, 116, 296, 116, 297, 116, 298, + 116, 300, 299, 301, 299, 302, 299, 303, + 299, 304, 299, 305, 116, 306, 116, 307, + 116, 308, 116, 309, 116, 310, 116, 311, + 116, 312, 116, 313, 116, 314, 116, 315, + 116, 316, 116, 317, 116, 318, 116, 319, + 116, 320, 116, 321, 116, 322, 116, 323, + 116, 324, 116, 325, 116, 326, 116, 327, + 116, 328, 116, 329, 330, 116, 331, 116, + 332, 116, 333, 116, 334, 116, 335, 116, + 336, 116, 337, 116, 338, 116, 339, 116, + 340, 116, 341, 116, 342, 116, 343, 116, + 344, 116, 345, 116, 346, 116, 347, 116, + 348, 116, 349, 116, 350, 351, 116, 352, + 116, 353, 116, 354, 116, 355, 116, 356, + 116, 357, 116, 358, 116, 359, 116, 360, + 116, 361, 116, 362, 116, 363, 116, 364, + 116, 365, 116, 366, 116, 367, 368, 369, + 370, 116, 371, 116, 372, 116, 373, 116, + 374, 116, 375, 116, 376, 116, 377, 116, + 378, 116, 379, 116, 380, 116, 381, 116, + 382, 116, 383, 116, 384, 116, 385, 116, + 386, 116, 387, 116, 388, 389, 116, 390, + 116, 391, 116, 392, 116, 393, 116, 394, + 116, 395, 116, 396, 116, 397, 116, 398, + 116, 400, 401, 399, 402, 399, 403, 399, + 404, 399, 405, 399, 406, 399, 407, 399, + 408, 409, 410, 399, 411, 399, 412, 399, + 413, 399, 414, 399, 415, 399, 416, 399, + 417, 399, 418, 399, 419, 420, 399, 421, + 399, 422, 399, 423, 399, 424, 399, 425, + 399, 426, 399, 428, 429, 427, 430, 427, + 431, 427, 432, 427, 433, 427, 434, 427, + 435, 427, 436, 427, 437, 427, 438, 427, + 439, 427, 441, 440, 442, 440, 443, 440, + 444, 440, 445, 440, 446, 440, 447, 440, + 448, 440, 449, 440, 450, 427, 451, 427, + 452, 427, 453, 427, 454, 427, 455, 427, + 456, 427, 457, 427, 458, 427, 459, 427, + 460, 427, 461, 427, 463, 462, 464, 462, + 465, 462, 466, 462, 467, 462, 468, 462, + 469, 462, 470, 462, 471, 462, 472, 462, + 473, 116, 474, 116, 475, 116, 476, 477, + 116, 478, 116, 479, 116, 480, 116, 481, + 116, 482, 116, 483, 116, 484, 485, 486, + 487, 116, 488, 116, 489, 116, 490, 116, + 491, 116, 492, 116, 493, 116, 494, 116, + 495, 116, 496, 116, 497, 116, 498, 116, + 499, 116, 500, 116, 501, 116, 502, 116, + 503, 116, 504, 116, 505, 116, 506, 116, + 507, 116, 508, 116, 509, 116, 510, 116, + 511, 116, 512, 116, 513, 116, 514, 116, + 515, 116, 516, 116, 517, 116, 518, 116, + 519, 116, 520, 116, 521, 116, 522, 116, + 523, 116, 525, 526, 524, 527, 524, 528, + 524, 529, 524, 530, 524, 531, 524, 532, + 524, 533, 524, 534, 524, 535, 524, 536, + 524, 537, 524, 538, 524, 539, 116, 540, + 116, 541, 116, 542, 116, 543, 116, 544, + 116, 545, 116, 547, 548, 546, 549, 546, + 550, 546, 551, 546, 552, 546, 553, 546, + 554, 546, 555, 546, 556, 546, 557, 546, + 558, 546, 559, 546, 560, 546, 561, 546, + 562, 546, 563, 546, 564, 546, 565, 546, + 566, 546, 567, 546, 568, 546, 569, 546, + 570, 546, 571, 546, 572, 546, 573, 546, + 574, 546, 575, 546, 576, 546, 577, 546, + 578, 546, 579, 580, 546, 581, 546, 582, + 546, 583, 546, 584, 546, 585, 546, 586, + 546, 587, 546, 588, 546, 589, 546, 590, + 546, 591, 546, 592, 546, 593, 594, 595, + 116, 596, 597, 116, 598, 116, 599, 116, + 600, 116, 601, 116, 602, 116, 603, 116, + 604, 116, 605, 116, 606, 607, 608, 116, + 609, 116, 610, 116, 611, 116, 612, 116, + 613, 116, 614, 116, 615, 116, 616, 116, + 617, 116, 618, 116, 619, 116, 620, 116, + 621, 116, 622, 116, 623, 624, 116, 625, + 116, 626, 116, 627, 628, 116, 629, 116, + 630, 116, 631, 116, 632, 116, 633, 116, + 634, 116, 635, 116, 636, 116, 637, 116, + 638, 116, 639, 116, 640, 116, 641, 116, + 642, 116, 643, 116, 644, 116, 645, 116, + 646, 116, 647, 116, 648, 116, 650, 649, + 652, 651, 653, 649, 649, 651, 656, 657, + 654, 655, 656, 657, 658, 655, 656, 657, + 655, 660, 661, 654, 662, 663, 664, 665, + 666, 667, 668, 669, 670, 671, 672, 673, + 659, 660, 661, 654, 659, 660, 661, 674, + 659, 660, 661, 659, 660, 661, 654, 675, + 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 659, 660, 661, 654, 686, 687, + 659, 660, 661, 654, 688, 689, 659, 660, + 661, 654, 690, 659, 660, 661, 654, 691, + 659, 692, 661, 654, 659, 660, 661, 693, + 659, 660, 661, 654, 694, 659, 660, 661, + 654, 695, 659, 696, 661, 654, 659, 660, + 661, 697, 659, 660, 661, 654, 698, 659, + 660, 661, 654, 699, 659, 660, 661, 654, + 700, 659, 701, 661, 654, 659, 660, 661, + 702, 659, 660, 661, 654, 703, 659, 660, + 661, 654, 704, 659, 660, 661, 654, 705, + 659, 660, 661, 654, 706, 659, 707, 661, + 654, 659, 660, 661, 708, 659, 660, 661, + 654, 709, 659, 660, 661, 654, 710, 659, + 660, 661, 654, 711, 659, 660, 661, 654, + 712, 659, 713, 661, 654, 659, 660, 661, + 714, 659, 660, 661, 654, 715, 659, 660, + 661, 654, 716, 659, 660, 661, 654, 717, + 659, 660, 661, 654, 718, 659, 719, 661, + 654, 659, 660, 661, 720, 659, 660, 661, + 654, 721, 659, 660, 661, 654, 722, 659, + 660, 661, 654, 723, 659, 660, 661, 654, + 724, 659, 725, 661, 654, 659, 660, 661, + 726, 659, 660, 661, 654, 727, 659, 660, + 661, 654, 728, 659, 660, 661, 654, 729, + 659, 660, 661, 654, 730, 659, 731, 661, + 654, 659, 660, 661, 732, 659, 660, 661, + 654, 733, 734, 659, 660, 661, 654, 735, + 659, 660, 661, 654, 736, 659, 660, 661, + 654, 737, 659, 738, 661, 654, 659, 660, + 661, 739, 659, 660, 661, 654, 740, 659, + 660, 661, 654, 741, 659, 660, 661, 654, + 742, 659, 743, 661, 654, 659, 660, 661, + 744, 659, 660, 661, 654, 745, 659, 660, + 661, 654, 746, 659, 660, 661, 654, 747, + 659, 660, 661, 654, 748, 659, 749, 661, + 654, 659, 660, 661, 750, 659, 660, 661, + 654, 751, 659, 660, 661, 654, 752, 659, + 660, 661, 654, 753, 659, 660, 661, 654, + 754, 659, 755, 661, 654, 659, 660, 661, + 756, 659, 660, 661, 654, 757, 659, 660, + 661, 654, 758, 659, 660, 661, 654, 759, + 659, 760, 661, 654, 659, 660, 661, 761, + 659, 660, 661, 654, 762, 659, 660, 661, + 654, 763, 659, 660, 661, 654, 764, 659, + 660, 661, 654, 765, 659, 660, 661, 654, + 766, 659, 767, 661, 654, 659, 660, 661, + 768, 659, 660, 661, 654, 769, 770, 659, + 660, 661, 654, 771, 772, 659, 660, 661, + 654, 773, 659, 660, 661, 654, 774, 659, + 775, 661, 654, 659, 660, 661, 776, 659, + 660, 661, 654, 777, 659, 660, 661, 654, + 778, 659, 779, 661, 654, 659, 660, 661, + 780, 659, 660, 661, 654, 781, 659, 660, + 661, 654, 782, 659, 660, 661, 654, 783, + 659, 784, 661, 654, 659, 660, 661, 785, + 659, 660, 661, 654, 786, 659, 660, 661, + 654, 787, 659, 660, 661, 654, 788, 659, + 660, 661, 654, 789, 659, 790, 661, 654, + 659, 660, 661, 791, 659, 660, 661, 654, + 792, 659, 660, 661, 654, 793, 659, 660, + 661, 654, 794, 659, 660, 661, 654, 795, + 659, 796, 661, 654, 659, 660, 661, 797, + 659, 660, 661, 654, 798, 659, 660, 661, + 654, 799, 659, 660, 661, 654, 800, 659, + 660, 661, 654, 801, 659, 802, 661, 654, + 659, 660, 661, 803, 659, 660, 661, 654, + 804, 659, 660, 661, 654, 805, 659, 660, + 661, 654, 806, 659, 660, 661, 654, 807, + 659, 808, 661, 654, 659, 660, 661, 809, + 659, 660, 661, 654, 810, 659, 660, 661, + 654, 811, 659, 660, 661, 654, 812, 659, + 660, 661, 654, 813, 659, 814, 661, 654, + 659, 660, 661, 815, 659, 660, 661, 654, + 816, 817, 659, 660, 661, 654, 818, 659, + 660, 661, 654, 819, 659, 660, 661, 654, + 820, 659, 821, 661, 654, 659, 660, 661, + 822, 659, 660, 661, 654, 823, 659, 660, + 661, 654, 824, 659, 660, 661, 654, 825, + 659, 826, 661, 654, 659, 660, 661, 827, + 659, 660, 661, 654, 828, 659, 660, 661, + 654, 829, 659, 660, 661, 654, 830, 659, + 660, 661, 654, 831, 659, 832, 661, 654, + 659, 660, 661, 833, 659, 660, 661, 654, + 834, 659, 660, 661, 654, 835, 659, 660, + 661, 654, 836, 659, 660, 661, 654, 837, + 659, 838, 661, 654, 659, 660, 661, 839, + 659, 660, 661, 654, 840, 659, 660, 661, + 654, 841, 659, 660, 661, 654, 842, 659, + 843, 661, 654, 659, 660, 661, 844, 659, + 660, 661, 654, 845, 659, 660, 661, 654, + 846, 659, 660, 661, 654, 847, 659, 660, + 661, 654, 848, 659, 660, 661, 654, 849, + 659, 850, 661, 654, 659, 660, 661, 851, + 659, 853, 854, 654, 852, 853, 854, 658, + 852, 853, 854, 852, 856, 855, 857, 856, + 855, 860, 859, 859, 859, 858, 862, 861, + 863, 861, 864, 861, 866, 865, 867, 865, + 868, 865, 870, 869, 871, 869, 872, 869, + 873, 876, 877, 878, 879, 880, 881, 882, + 883, 884, 885, 886, 887, 888, 875, 893, + 875, 876, 875, 875, 889, 890, 891, 892, + 889, 874, 895, 896, 894, 23, 23, 23, + 23, 897, 25, 25, 25, 25, 897, 899, + 30, 902, 903, 904, 30, 900, 901, 30, + 30, 898, 44, 897, 47, 47, 47, 47, + 44, 897, 43, 44, 897, 905, 906, 48, + 48, 48, 48, 897, 907, 897, 49, 50, + 897, 908, 909, 910, 897, 45, 53, 46, + 54, 54, 54, 54, 897, 912, 913, 911, + 915, 916, 914, 918, 919, 917, 56, 60, + 64, 920, 923, 926, 927, 928, 929, 930, + 931, 932, 933, 934, 934, 935, 936, 937, + 938, 934, 939, 940, 941, 942, 943, 944, + 945, 946, 947, 948, 949, 950, 951, 934, + 952, 953, 954, 955, 956, 957, 934, 958, + 959, 960, 961, 924, 925, 922, 963, 962, + 964, 962, 966, 967, 965, 969, 967, 968, + 967, 970, 973, 972, 975, 68, 977, 71, + 979, 978, 976, 981, 980, 982, 980, 984, + 983, 985, 983, 987, 988, 989, 986, 991, + 990, 994, 993, 999, 996, 997, 998, 995, + 1000, 1001, 1002, 995, 94, 94, 94, 1003, + 98, 1004, 1006, 1007, 1005, 1009, 1008, 1010, + 1008, 1011, 1008, 1013, 1014, 1012, 109, 108, + 109, 1016, 1017, 108, 1019, 1020, 1021, 1022, + 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, + 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, + 1039, 1040, 1041, 1018, 1043, 1044, 1045, 1042, + 1046, 1047, 1048, 1049, 1050, 1042, 1052, 1053, + 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1051, + 1062, 1063, 1061, 1064, 1042, 1065, 1066, 1042, + 1067, 1068, 1069, 1070, 1071, 1042, 1072, 1073, + 1074, 1042, 1076, 1077, 1075, 1078, 1079, 1042, + 1080, 1042, 1081, 1082, 1042, 1084, 1085, 1086, + 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1083, + 1095, 1096, 1097, 1098, 1099, 1100, 1094, 1102, + 1101, 1104, 1105, 1106, 1107, 1108, 1103, 1109, + 1110, 1111, 1112, 1042, 1114, 1115, 1116, 1117, + 1118, 1119, 1120, 1121, 1113, 1122, 1123, 1042, + 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, + 1133, 1124, 1134, 1135, 1136, 1137, 1042, 1138, + 1042, 1139, 1042, 1140, 1141, 1142, 1143, 1042, + 1144, 1042, 1146, 1147, 1148, 1145, 649, 1150, + 1151, 1152, 1153, 1154, 1155, 1156, 1149, 1158, + 1159, 1160, 1161, 1157, 1162, 1163, 1164, 1165, + 1162, 874, 655, 1167, 852, 1166, 1169, 1173, + 1174, 1175, 1176, 1176, 1177, 1178, 1179, 1176, + 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, + 1188, 1189, 1176, 1190, 1191, 1192, 1193, 1194, + 1195, 1176, 1196, 1197, 1198, 1170, 1171, 1172, + 1172, 1168, 1200, 1199, 1201, 1199, 1203, 1202, + 1204, 1202, 1207, 1206, 1209, 1211, 1210, 1214, + 1213, 1219, 1216, 1217, 1218, 1215, 1220, 1221, + 1222, 1215, 859, 859, 859, 1223, 1225, 1224, + 1226, 1224, 1227, 1224, 1229, 1230, 1231, 1228, + 1228, 1228, 874, 1233, 1234, 1232, 1236, 1235, + 1237, 1238, 1239, 1240, 1237, 874, 1242, 1241, + 1244, 1243, 1245, 1243, 1246, 1243, 1248, 1247, + 1249, 1250, 1251, 1252, 1249, 874, 1254, 1253, + 1256, 1255, 1257, 1255, 1258, 1255, 1260, 1259, + 1262, 1261, 0 +}; + +static const short _regex_trans_targs[] = { + 746, 746, 746, 746, 746, 748, 749, 750, + 746, 751, 752, 753, 746, 754, 746, 746, + 755, 756, 757, 758, 746, 746, 746, 3, + 746, 4, 746, 6, 7, 746, 8, 746, + 9, 12, 746, 14, 746, 746, 746, 16, + 746, 18, 17, 746, 19, 746, 746, 20, + 21, 746, 22, 25, 746, 27, 28, 746, + 29, 30, 31, 746, 32, 33, 34, 746, + 35, 36, 37, 746, 38, 746, 772, 40, + 42, 46, 49, 43, 44, 746, 45, 47, + 746, 48, 746, 746, 51, 746, 53, 746, + 55, 746, 746, 57, 746, 746, 58, 746, + 746, 60, 59, 783, 61, 783, 783, 746, + 746, 64, 746, 787, 65, 787, 67, 787, + 69, 787, 70, 787, 790, 790, 73, 76, + 74, 75, 790, 77, 78, 79, 80, 790, + 82, 83, 84, 85, 790, 87, 92, 94, + 88, 89, 90, 91, 790, 93, 790, 95, + 790, 97, 98, 99, 100, 790, 102, 103, + 104, 105, 106, 790, 108, 109, 111, 110, + 790, 112, 113, 790, 115, 120, 116, 117, + 118, 119, 790, 121, 790, 790, 123, 139, + 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 790, + 140, 141, 790, 143, 144, 790, 145, 146, + 147, 148, 790, 790, 150, 151, 790, 153, + 154, 790, 156, 157, 158, 159, 160, 161, + 790, 163, 167, 164, 165, 166, 790, 168, + 169, 170, 171, 790, 173, 177, 174, 175, + 176, 790, 178, 179, 180, 181, 182, 183, + 790, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 790, 203, 204, 205, 206, 207, + 790, 209, 210, 211, 212, 213, 790, 215, + 216, 217, 218, 219, 220, 221, 790, 223, + 224, 225, 790, 227, 228, 790, 230, 235, + 231, 232, 233, 234, 790, 236, 237, 238, + 239, 790, 799, 790, 242, 790, 244, 245, + 790, 247, 248, 249, 790, 251, 252, 253, + 254, 255, 790, 257, 258, 259, 260, 261, + 262, 263, 264, 265, 266, 267, 268, 269, + 790, 271, 277, 272, 273, 274, 275, 276, + 790, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 295, + 292, 293, 294, 790, 296, 297, 298, 299, + 790, 301, 302, 303, 304, 305, 790, 307, + 310, 314, 319, 308, 309, 790, 311, 312, + 313, 790, 315, 316, 317, 318, 790, 320, + 321, 322, 323, 790, 325, 332, 326, 327, + 328, 329, 330, 331, 790, 333, 790, 790, + 790, 335, 336, 790, 338, 339, 340, 790, + 342, 344, 349, 343, 790, 345, 346, 347, + 348, 790, 790, 351, 354, 352, 353, 790, + 355, 356, 790, 790, 358, 364, 359, 360, + 361, 362, 363, 790, 365, 366, 367, 790, + 790, 369, 370, 371, 372, 373, 374, 375, + 376, 790, 378, 379, 380, 381, 382, 383, + 790, 385, 386, 387, 388, 790, 790, 390, + 391, 392, 393, 394, 395, 396, 397, 790, + 790, 400, 401, 790, 403, 408, 404, 405, + 406, 407, 790, 409, 410, 415, 421, 433, + 411, 412, 413, 414, 790, 416, 417, 418, + 419, 420, 790, 422, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 790, 434, + 435, 436, 437, 790, 439, 440, 790, 442, + 443, 444, 445, 790, 790, 447, 452, 448, + 449, 450, 451, 790, 453, 454, 455, 456, + 457, 458, 790, 460, 461, 462, 790, 464, + 465, 790, 790, 467, 473, 468, 469, 470, + 471, 472, 790, 474, 475, 476, 477, 478, + 479, 790, 481, 482, 483, 484, 790, 486, + 487, 488, 489, 790, 491, 492, 493, 494, + 495, 496, 790, 498, 507, 499, 500, 501, + 502, 503, 504, 505, 506, 790, 508, 509, + 790, 511, 519, 528, 512, 515, 513, 514, + 790, 516, 517, 518, 790, 520, 521, 522, + 525, 790, 523, 524, 790, 526, 527, 790, + 529, 790, 531, 532, 533, 790, 535, 536, + 790, 537, 790, 539, 543, 540, 541, 542, + 790, 544, 545, 546, 547, 790, 549, 550, + 551, 552, 553, 790, 790, 790, 790, 790, + 790, 0, 560, 561, 562, 817, 819, 563, + 564, 565, 819, 567, 568, 569, 570, 651, + 666, 672, 678, 684, 690, 696, 707, 713, + 719, 724, 819, 571, 586, 592, 598, 604, + 610, 616, 627, 633, 639, 644, 572, 581, + 573, 577, 574, 575, 576, 819, 578, 579, + 580, 819, 582, 583, 584, 585, 819, 587, + 588, 589, 590, 591, 819, 593, 594, 595, + 596, 597, 819, 599, 600, 601, 602, 603, + 819, 605, 606, 607, 608, 609, 819, 611, + 612, 613, 614, 615, 819, 617, 622, 618, + 619, 620, 621, 819, 623, 624, 625, 626, + 819, 628, 629, 630, 631, 632, 819, 634, + 635, 636, 637, 638, 819, 640, 641, 642, + 643, 819, 645, 646, 647, 648, 649, 650, + 819, 652, 661, 653, 657, 654, 655, 656, + 819, 658, 659, 660, 819, 662, 663, 664, + 665, 819, 667, 668, 669, 670, 671, 819, + 673, 674, 675, 676, 677, 819, 679, 680, + 681, 682, 683, 819, 685, 686, 687, 688, + 689, 819, 691, 692, 693, 694, 695, 819, + 697, 702, 698, 699, 700, 701, 819, 703, + 704, 705, 706, 819, 708, 709, 710, 711, + 712, 819, 714, 715, 716, 717, 718, 819, + 720, 721, 722, 723, 819, 725, 726, 727, + 728, 729, 730, 819, 731, 732, 733, 819, + 735, 819, 819, 736, 819, 819, 819, 739, + 819, 838, 838, 742, 838, 843, 843, 745, + 843, 746, 0, 746, 746, 746, 747, 746, + 759, 760, 746, 761, 762, 763, 746, 782, + 746, 746, 784, 785, 786, 746, 746, 1, + 2, 746, 746, 5, 9, 10, 11, 13, + 15, 746, 746, 746, 23, 24, 26, 746, + 746, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 764, 766, 768, 746, 746, + 746, 746, 746, 746, 746, 746, 746, 769, + 746, 746, 746, 746, 746, 746, 746, 746, + 746, 770, 746, 746, 746, 771, 746, 776, + 746, 777, 778, 746, 746, 746, 746, 746, + 779, 746, 746, 765, 746, 746, 767, 768, + 746, 768, 746, 746, 746, 746, 746, 746, + 746, 39, 774, 41, 746, 773, 746, 746, + 775, 746, 746, 50, 52, 54, 746, 56, + 746, 746, 746, 746, 780, 780, 780, 781, + 746, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 62, 63, 788, 787, 789, 787, + 66, 68, 790, 791, 792, 793, 795, 796, + 797, 798, 800, 801, 802, 803, 804, 806, + 807, 808, 809, 810, 811, 812, 813, 814, + 815, 816, 790, 71, 72, 81, 86, 96, + 101, 107, 114, 790, 122, 790, 790, 142, + 790, 794, 790, 155, 162, 790, 149, 152, + 172, 184, 202, 208, 214, 222, 226, 229, + 240, 246, 250, 790, 241, 243, 256, 270, + 300, 306, 324, 790, 790, 334, 337, 341, + 790, 790, 790, 790, 790, 350, 790, 357, + 790, 805, 790, 377, 384, 790, 368, 790, + 790, 389, 398, 790, 790, 399, 402, 438, + 441, 790, 790, 790, 790, 790, 446, 790, + 790, 790, 459, 463, 790, 466, 790, 480, + 485, 790, 790, 790, 490, 497, 510, 530, + 534, 538, 548, 554, 555, 556, 557, 558, + 790, 790, 790, 790, 790, 818, 818, 818, + 818, 818, 818, 818, 818, 819, 819, 820, + 821, 819, 819, 833, 834, 835, 819, 566, + 819, 822, 824, 819, 819, 819, 819, 819, + 819, 826, 819, 819, 819, 819, 819, 819, + 827, 819, 819, 819, 819, 819, 819, 828, + 829, 819, 819, 819, 819, 819, 830, 819, + 823, 819, 819, 825, 819, 819, 819, 819, + 819, 819, 819, 734, 819, 819, 819, 819, + 831, 831, 831, 832, 819, 819, 819, 819, + 819, 819, 737, 738, 836, 837, 836, 836, + 836, 836, 836, 838, 839, 838, 840, 841, + 842, 838, 838, 838, 838, 740, 741, 843, + 844, 843, 845, 846, 847, 843, 843, 843, + 843, 743, 744, 848, 848, 849, 849 +}; + +static const short _regex_trans_actions[] = { + 827, 631, 765, 731, 723, 45, 903, 903, + 897, 45, 912, 45, 900, 903, 729, 741, + 0, 45, 45, 923, 737, 841, 747, 0, + 743, 3, 839, 3, 0, 761, 3, 759, + 870, 3, 751, 0, 749, 755, 753, 0, + 757, 3, 0, 745, 0, 725, 727, 27, + 3, 763, 0, 3, 673, 0, 25, 829, + 0, 0, 0, 603, 0, 0, 0, 601, + 0, 0, 0, 831, 0, 675, 17, 0, + 7, 870, 3, 17, 17, 663, 17, 870, + 661, 870, 665, 837, 3, 671, 3, 669, + 3, 667, 833, 0, 677, 835, 0, 679, + 845, 0, 11, 29, 13, 31, 0, 843, + 769, 0, 771, 59, 0, 53, 0, 51, + 0, 49, 0, 47, 359, 315, 0, 0, + 0, 0, 127, 0, 0, 0, 0, 129, + 0, 0, 0, 0, 131, 0, 0, 0, + 0, 0, 0, 0, 133, 0, 135, 0, + 137, 0, 0, 0, 0, 139, 0, 0, + 0, 0, 0, 141, 0, 0, 0, 0, + 143, 0, 0, 145, 0, 0, 0, 0, + 0, 0, 147, 0, 149, 341, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 151, + 0, 0, 153, 0, 0, 155, 0, 0, + 0, 0, 157, 343, 0, 0, 159, 0, + 0, 161, 0, 0, 0, 0, 0, 0, + 163, 0, 0, 0, 0, 0, 165, 0, + 0, 0, 0, 167, 0, 0, 0, 0, + 0, 169, 0, 0, 0, 0, 0, 0, + 171, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 173, 0, 0, 0, 0, 0, + 175, 0, 0, 0, 0, 0, 177, 0, + 0, 0, 0, 0, 0, 0, 179, 0, + 0, 0, 181, 0, 0, 183, 0, 0, + 0, 0, 0, 0, 185, 0, 0, 0, + 0, 187, 45, 357, 0, 189, 0, 0, + 191, 0, 0, 0, 193, 0, 0, 0, + 0, 0, 195, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 0, 0, 0, 0, + 199, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 201, 0, 0, 0, 0, + 203, 0, 0, 0, 0, 0, 205, 0, + 0, 0, 0, 0, 0, 207, 0, 0, + 0, 209, 0, 0, 0, 0, 211, 0, + 0, 0, 0, 213, 0, 0, 0, 0, + 0, 0, 0, 0, 215, 0, 217, 345, + 219, 0, 0, 221, 0, 0, 0, 223, + 0, 0, 0, 0, 225, 0, 0, 0, + 0, 227, 229, 0, 0, 0, 0, 231, + 0, 0, 233, 347, 0, 0, 0, 0, + 0, 0, 0, 235, 0, 0, 0, 237, + 349, 0, 0, 0, 0, 0, 0, 0, + 0, 239, 0, 0, 0, 0, 0, 0, + 241, 0, 0, 0, 0, 243, 351, 0, + 0, 0, 0, 0, 0, 0, 0, 245, + 247, 0, 0, 249, 0, 0, 0, 0, + 0, 0, 251, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 253, 0, 0, 0, + 0, 0, 255, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 257, 0, + 0, 0, 0, 259, 0, 0, 261, 0, + 0, 0, 0, 263, 353, 0, 0, 0, + 0, 0, 0, 265, 0, 0, 0, 0, + 0, 0, 267, 0, 0, 0, 269, 0, + 0, 271, 355, 0, 0, 0, 0, 0, + 0, 0, 273, 0, 0, 0, 0, 0, + 0, 275, 0, 0, 0, 0, 277, 0, + 0, 0, 0, 279, 0, 0, 0, 0, + 0, 0, 281, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 283, 0, 0, + 285, 0, 0, 0, 0, 0, 0, 0, + 287, 0, 0, 0, 289, 0, 0, 0, + 0, 291, 0, 0, 293, 0, 0, 295, + 0, 297, 0, 0, 0, 299, 0, 0, + 303, 0, 301, 0, 0, 0, 0, 0, + 305, 0, 0, 0, 0, 307, 0, 0, + 0, 0, 0, 309, 311, 119, 121, 123, + 125, 39, 0, 35, 33, 37, 539, 0, + 0, 0, 377, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 435, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 381, 0, 0, + 0, 385, 0, 0, 0, 0, 389, 0, + 0, 0, 0, 0, 393, 0, 0, 0, + 0, 0, 397, 0, 0, 0, 0, 0, + 401, 0, 0, 0, 0, 0, 405, 0, + 0, 0, 0, 0, 409, 0, 0, 0, + 0, 0, 0, 413, 0, 0, 0, 0, + 417, 0, 0, 0, 0, 0, 421, 0, + 0, 0, 0, 0, 425, 0, 0, 0, + 0, 429, 0, 0, 0, 0, 0, 0, + 433, 0, 0, 0, 0, 0, 0, 0, + 379, 0, 0, 0, 383, 0, 0, 0, + 0, 387, 0, 0, 0, 0, 0, 391, + 0, 0, 0, 0, 0, 395, 0, 0, + 0, 0, 0, 399, 0, 0, 0, 0, + 0, 403, 0, 0, 0, 0, 0, 407, + 0, 0, 0, 0, 0, 0, 411, 0, + 0, 0, 0, 415, 0, 0, 0, 0, + 0, 419, 0, 0, 0, 0, 0, 423, + 0, 0, 0, 0, 427, 0, 0, 0, + 0, 0, 0, 431, 0, 0, 0, 533, + 0, 471, 535, 0, 475, 537, 503, 0, + 505, 569, 557, 0, 559, 587, 575, 0, + 577, 633, 0, 777, 775, 637, 45, 597, + 0, 0, 609, 0, 45, 0, 635, 909, + 599, 773, 0, 45, 45, 629, 779, 0, + 0, 821, 819, 1, 855, 855, 1, 0, + 3, 735, 733, 739, 1, 1, 0, 783, + 615, 613, 785, 619, 617, 787, 623, 621, + 781, 817, 721, 5, 852, 915, 639, 647, + 611, 695, 607, 717, 699, 715, 683, 0, + 605, 713, 691, 703, 687, 719, 641, 657, + 645, 0, 693, 659, 655, 906, 697, 45, + 651, 45, 0, 653, 689, 649, 701, 685, + 7, 643, 791, 15, 867, 795, 858, 919, + 793, 927, 847, 811, 711, 709, 809, 681, + 801, 7, 17, 849, 799, 17, 876, 797, + 17, 873, 815, 1, 1, 1, 803, 0, + 813, 707, 705, 805, 19, 23, 21, 45, + 882, 894, 888, 807, 825, 789, 627, 625, + 823, 767, 0, 0, 45, 55, 45, 57, + 0, 0, 317, 45, 45, 45, 45, 45, + 45, 45, 45, 45, 45, 45, 45, 45, + 45, 45, 45, 45, 45, 45, 45, 45, + 0, 0, 339, 0, 0, 0, 0, 0, + 0, 0, 0, 319, 0, 61, 63, 0, + 65, 45, 67, 0, 0, 321, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 337, 0, 0, 0, 0, + 0, 0, 0, 323, 79, 0, 0, 0, + 69, 71, 73, 75, 77, 0, 325, 0, + 81, 45, 83, 0, 0, 327, 0, 329, + 85, 0, 0, 87, 89, 0, 0, 0, + 0, 331, 91, 93, 95, 97, 0, 99, + 101, 103, 0, 0, 333, 0, 105, 0, + 0, 107, 109, 111, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 313, 335, 113, 115, 117, 375, 361, 363, + 365, 367, 369, 371, 373, 509, 491, 45, + 0, 511, 507, 0, 45, 45, 531, 0, + 499, 5, 9, 473, 497, 489, 439, 457, + 493, 0, 437, 485, 461, 481, 451, 441, + 0, 487, 453, 449, 495, 455, 445, 45, + 0, 447, 483, 443, 459, 479, 7, 517, + 15, 861, 519, 15, 864, 513, 469, 467, + 527, 477, 521, 0, 515, 465, 463, 523, + 19, 23, 21, 45, 879, 891, 885, 525, + 529, 501, 0, 0, 549, 0, 543, 541, + 551, 547, 545, 563, 0, 561, 0, 45, + 45, 567, 553, 565, 555, 0, 0, 581, + 0, 579, 0, 45, 45, 585, 571, 583, + 573, 0, 0, 591, 589, 595, 593 +}; + +static const short _regex_to_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 41, + 0, 41, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 41, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 41, 0, 0, 41, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 41, 41, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 41, 0, 41, 0, + 0, 0, 0, 41, 0, 0, 0, 0, + 41, 41 +}; + +static const short _regex_from_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 43, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 43, 0, 0, 43, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 43, 43, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 43, 0, 43, 0, + 0, 0, 0, 43, 0, 0, 0, 0, + 43, 43 +}; + +static const short _regex_eof_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 39, + 39, 39, 39, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0 +}; + +static const short _regex_eof_trans[] = { + 0, 1, 1, 22, 22, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 84, 84, 84, 84, 84, 84, + 91, 91, 94, 97, 97, 97, 104, 104, + 104, 108, 108, 108, 108, 108, 108, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 204, 204, 204, + 204, 204, 204, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 300, 300, 300, 300, 300, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, + 441, 441, 441, 441, 441, 441, 441, 441, + 441, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 463, 463, 463, + 463, 463, 463, 463, 463, 463, 463, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 525, 525, + 525, 525, 525, 525, 525, 525, 525, 525, + 525, 525, 525, 117, 117, 117, 117, 117, + 117, 117, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 0, + 0, 0, 0, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 856, 856, + 859, 862, 862, 862, 866, 866, 866, 870, + 870, 870, 0, 895, 898, 898, 899, 898, + 898, 898, 898, 898, 898, 898, 898, 912, + 915, 918, 921, 922, 963, 963, 966, 969, + 971, 972, 975, 977, 981, 981, 984, 984, + 987, 991, 993, 996, 996, 1004, 1005, 1006, + 1009, 1009, 1009, 0, 1016, 1016, 0, 1043, + 1043, 1052, 1062, 1043, 1043, 1043, 1043, 1076, + 1043, 1043, 1043, 1084, 1095, 1102, 1104, 1043, + 1114, 1043, 1125, 1043, 1043, 1043, 1043, 1043, + 1146, 0, 0, 0, 1167, 1167, 1200, 1200, + 1203, 1203, 1206, 1209, 1211, 1213, 1216, 1216, + 1224, 1225, 1225, 1225, 0, 1233, 0, 1242, + 1244, 1244, 1244, 0, 1254, 1256, 1256, 1256, + 0, 0 +}; + +static const int regex_start = 746; +static const int regex_error = 0; + +static const int regex_en_readVerb = 787; +static const int regex_en_readUCP = 790; +static const int regex_en_readBracedUCP = 559; +static const int regex_en_readUCPSingle = 818; +static const int regex_en_charClassGuts = 819; +static const int regex_en_readClass = 836; +static const int regex_en_readQuotedLiteral = 838; +static const int regex_en_readQuotedClass = 843; +static const int regex_en_readComment = 848; +static const int regex_en_readNewlineTerminatedComment = 849; +static const int regex_en_main = 746; + + +#line 1912 "Parser.rl" + +/** \brief Main parser call, returns root Component or nullptr. */ +unique_ptr parse(const char *ptr, ParseMode &globalMode) { + assert(ptr); + + const char *p = ptr; + const char *pe = ptr + strlen(ptr); + + // First, read the control verbs, set any global mode flags and move the + // ptr forward. + p = read_control_verbs(p, pe, 0, globalMode); + + const char *eof = pe; + int cs; + UNUSED int act; + int top; + vector stack; + const char *ts, *te; + unichar accumulator = 0; + unichar octAccumulator = 0; /* required as we are also accumulating for + * back ref when looking for octals */ + unsigned repeatN = 0; + unsigned repeatM = 0; + string label; + + ParseMode mode = globalMode; + ParseMode newMode; + + bool negated = false; + bool inComment = false; + + // Stack of sequences and flags used to store state when we enter + // sub-sequences. + vector sequences; + + // Index of the next capturing group. Note that zero is reserved for the + // root sequence. + unsigned groupIndex = 1; + + // Set storing group names that are currently in use. + flat_set groupNames; + + // Root sequence. + unique_ptr rootSeq = std::make_unique(); + rootSeq->setCaptureIndex(0); + + // Current sequence being appended to + ComponentSequence *currentSeq = rootSeq.get(); + + // The current character class being appended to. This is used as the + // accumulator for both character class and UCP properties. + unique_ptr currentCls; + + // True if the machine is currently inside a character class, i.e. square + // brackets [..]. + bool inCharClass = false; + + // True if the machine is inside a character class but it has not processed + // any "real" elements yet, i.e. it's still processing meta-characters like + // '^'. + bool inCharClassEarly = false; + + // Location at which the current character class began. + const char *currentClsBegin = p; + + // We throw exceptions on various parsing failures beyond this point: we + // use a try/catch block here to clean up our allocated memory before we + // re-throw the exception to the caller. + try { + // Embed the Ragel machine here + +#line 2533 "Parser.cpp" + { + cs = regex_start; + top = 0; + ts = 0; + te = 0; + act = 0; + } + +#line 1983 "Parser.rl" + +#line 2544 "Parser.cpp" + { + int _klen; + unsigned int _trans; + short _widec; + const short *_acts; + unsigned int _nacts; + const short *_keys; + + if ( p == pe ) + goto _test_eof; + if ( cs == 0 ) + goto _out; +_resume: + _acts = _regex_actions + _regex_from_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 24: +#line 1 "NONE" + {ts = p;} + break; +#line 2566 "Parser.cpp" + } + } + + _widec = (*p); + _klen = _regex_cond_lengths[cs]; + _keys = _regex_cond_keys + (_regex_cond_offsets[cs]*2); + if ( _klen > 0 ) { + const short *_lower = _keys; + const short *_mid; + const short *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( _widec < _mid[0] ) + _upper = _mid - 2; + else if ( _widec > _mid[1] ) + _lower = _mid + 2; + else { + switch ( _regex_cond_spaces[_regex_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { + case 0: { + _widec = (short)(128 + ((*p) - -128)); + if ( +#line 475 "Parser.rl" + mode.utf8 ) _widec += 256; + break; + } + case 1: { + _widec = (short)(1152 + ((*p) - -128)); + if ( +#line 476 "Parser.rl" + mode.ignore_space ) _widec += 256; + break; + } + case 2: { + _widec = (short)(640 + ((*p) - -128)); + if ( +#line 477 "Parser.rl" + inCharClassEarly ) _widec += 256; + break; + } + } + break; + } + } + } + + _keys = _regex_trans_keys + _regex_key_offsets[cs]; + _trans = _regex_index_offsets[cs]; + + _klen = _regex_single_lengths[cs]; + if ( _klen > 0 ) { + const short *_lower = _keys; + const short *_mid; + const short *_upper = _keys + _klen - 1; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + ((_upper-_lower) >> 1); + if ( _widec < *_mid ) + _upper = _mid - 1; + else if ( _widec > *_mid ) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); + goto _match; + } + } + _keys += _klen; + _trans += _klen; + } + + _klen = _regex_range_lengths[cs]; + if ( _klen > 0 ) { + const short *_lower = _keys; + const short *_mid; + const short *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( _widec < _mid[0] ) + _upper = _mid - 2; + else if ( _widec > _mid[1] ) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys)>>1); + goto _match; + } + } + _trans += _klen; + } + +_match: + _trans = _regex_indicies[_trans]; +_eof_trans: + cs = _regex_trans_targs[_trans]; + + if ( _regex_trans_actions[_trans] == 0 ) + goto _again; + + _acts = _regex_actions + _regex_trans_actions[_trans]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) + { + switch ( *_acts++ ) + { + case 0: +#line 285 "Parser.rl" + { label.clear();} + break; + case 1: +#line 286 "Parser.rl" + { label.push_back((*p));} + break; + case 2: +#line 287 "Parser.rl" + { octAccumulator = 0;} + break; + case 3: +#line 288 "Parser.rl" + { accumulator = 0;} + break; + case 4: +#line 289 "Parser.rl" + { + octAccumulator = 0; + pushOct(&octAccumulator, (*p)); + } + break; + case 5: +#line 293 "Parser.rl" + { + accumulator = 0; + pushDec(&accumulator, (*p)); + } + break; + case 6: +#line 297 "Parser.rl" + { repeatN = 0; repeatM = 0; } + break; + case 7: +#line 298 "Parser.rl" + { pushDec(&repeatN, (*p)); } + break; + case 8: +#line 299 "Parser.rl" + { pushDec(&repeatM, (*p)); } + break; + case 9: +#line 300 "Parser.rl" + { pushOct(&octAccumulator, (*p)); } + break; + case 10: +#line 301 "Parser.rl" + { pushDec(&accumulator, (*p)); } + break; + case 11: +#line 302 "Parser.rl" + { + accumulator *= 16; + accumulator += (*p) - '0'; + } + break; + case 12: +#line 306 "Parser.rl" + { + accumulator *= 16; + accumulator += 10 + (*p) - 'a'; + } + break; + case 13: +#line 310 "Parser.rl" + { + accumulator *= 16; + accumulator += 10 + (*p) - 'A'; + } + break; + case 14: +#line 430 "Parser.rl" + { + newMode = mode; + } + break; + case 15: +#line 437 "Parser.rl" + { + switch ((*p)) { + case 'i': + newMode.caseless = true; + break; + case 'm': + newMode.multiline = true; + break; + case 's': + newMode.dotall = true; + break; + case 'x': + newMode.ignore_space = true; + break; + default: + assert(0); // this action only called for [imsx] + break; + } + } + break; + case 16: +#line 456 "Parser.rl" + { + switch ((*p)) { + case 'i': + newMode.caseless = false; + break; + case 'm': + newMode.multiline = false; + break; + case 's': + newMode.dotall = false; + break; + case 'x': + newMode.ignore_space = false; + break; + default: + assert(0); // this action only called for [imsx] + break; + } + } + break; + case 17: +#line 510 "Parser.rl" + {repeatM = repeatN;} + break; + case 18: +#line 510 "Parser.rl" + {repeatM = ComponentRepeat::NoLimit;} + break; + case 19: +#line 722 "Parser.rl" + { negated = !negated; } + break; + case 20: +#line 723 "Parser.rl" + { p--; { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 790;goto _again;}} } + break; + case 21: +#line 724 "Parser.rl" + { if (!inCharClass) { // not inside [..] + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + } + break; + case 22: +#line 730 "Parser.rl" + { throw LocatedParseError("Malformed property"); } + break; + case 25: +#line 1 "NONE" + {te = p+1;} + break; + case 26: +#line 550 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("(*UTF8) must be at start of " + "expression, encountered"); + }} + break; + case 27: +#line 554 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("(*UTF) must be at start of " + "expression, encountered"); + }} + break; + case 28: +#line 558 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("(*UCP) must be at start of " + "expression, encountered"); + }} + break; + case 29: +#line 564 "Parser.rl" + {te = p+1;{ + ParseMode temp_mode; + assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. + read_control_verbs(ts - 2, te, (ts - 2 - ptr), temp_mode); + assert(0); // Should have thrown a parse error. + throw LocatedParseError("Unknown control verb"); + }} + break; + case 30: +#line 571 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unknown control verb"); + }} + break; + case 31: +#line 571 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Unknown control verb"); + }} + break; + case 32: +#line 571 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Unknown control verb"); + }} + break; + case 33: +#line 581 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 34: +#line 582 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CF, negated); {cs = stack[--top]; goto _again;} }} + break; + case 35: +#line 583 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 36: +#line 585 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 37: +#line 587 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 38: +#line 588 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 39: +#line 589 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 40: +#line 590 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LT, negated); {cs = stack[--top]; goto _again;} }} + break; + case 41: +#line 591 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 42: +#line 592 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_L_AND, negated); {cs = stack[--top]; goto _again;} }} + break; + case 43: +#line 594 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_MC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 44: +#line 596 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_MN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 45: +#line 598 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ND, negated); {cs = stack[--top]; goto _again;} }} + break; + case 46: +#line 599 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_NL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 47: +#line 600 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_NO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 48: +#line 602 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 49: +#line 603 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PD, negated); {cs = stack[--top]; goto _again;} }} + break; + case 50: +#line 604 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 51: +#line 605 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PF, negated); {cs = stack[--top]; goto _again;} }} + break; + case 52: +#line 606 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 53: +#line 607 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 54: +#line 608 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 55: +#line 610 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 56: +#line 611 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 57: +#line 612 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 58: +#line 613 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 59: +#line 615 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ZL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 60: +#line 616 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ZP, negated); {cs = stack[--top]; goto _again;} }} + break; + case 61: +#line 617 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ZS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 62: +#line 618 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 63: +#line 619 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XPS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 64: +#line 620 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XSP, negated); {cs = stack[--top]; goto _again;} }} + break; + case 65: +#line 621 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XWD, negated); {cs = stack[--top]; goto _again;} }} + break; + case 66: +#line 622 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ARABIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 67: +#line 623 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 68: +#line 624 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 69: +#line 625 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BALINESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 70: +#line 626 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BAMUM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 71: +#line 627 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BATAK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 72: +#line 628 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BENGALI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 73: +#line 629 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 74: +#line 630 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 75: +#line 631 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 76: +#line 632 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 77: +#line 633 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BUHID, negated); {cs = stack[--top]; goto _again;} }} + break; + case 78: +#line 634 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 79: +#line 635 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CARIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 80: +#line 636 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CHAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 81: +#line 637 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 82: +#line 638 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_COMMON, negated); {cs = stack[--top]; goto _again;} }} + break; + case 83: +#line 639 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_COPTIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 84: +#line 640 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 85: +#line 641 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {cs = stack[--top]; goto _again;} }} + break; + case 86: +#line 642 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 87: +#line 643 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_DESERET, negated); {cs = stack[--top]; goto _again;} }} + break; + case 88: +#line 644 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 89: +#line 645 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 90: +#line 646 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 91: +#line 647 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 92: +#line 648 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 93: +#line 649 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 94: +#line 650 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GREEK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 95: +#line 651 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 96: +#line 652 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 97: +#line 654 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HANGUL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 98: +#line 655 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 99: +#line 656 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HEBREW, negated); {cs = stack[--top]; goto _again;} }} + break; + case 100: +#line 657 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 101: +#line 658 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 102: +#line 659 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_INHERITED, negated); {cs = stack[--top]; goto _again;} }} + break; + case 103: +#line 660 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 104: +#line 661 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 105: +#line 662 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 106: +#line 663 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KAITHI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 107: +#line 664 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KANNADA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 108: +#line 665 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 109: +#line 666 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 110: +#line 667 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 111: +#line 668 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KHMER, negated); {cs = stack[--top]; goto _again;} }} + break; + case 112: +#line 669 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LAO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 113: +#line 670 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LATIN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 114: +#line 671 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 115: +#line 672 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LIMBU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 116: +#line 673 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {cs = stack[--top]; goto _again;} }} + break; + case 117: +#line 674 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LISU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 118: +#line 675 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 119: +#line 676 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 120: +#line 677 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 121: +#line 678 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 122: +#line 679 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 123: +#line 680 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 124: +#line 681 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {cs = stack[--top]; goto _again;} }} + break; + case 125: +#line 682 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 126: +#line 683 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_NKO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 127: +#line 684 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OGHAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 128: +#line 685 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 129: +#line 686 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 130: +#line 687 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 131: +#line 688 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 132: +#line 689 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 133: +#line 690 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ORIYA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 134: +#line 691 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 135: +#line 692 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 136: +#line 693 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 137: +#line 694 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_REJANG, negated); {cs = stack[--top]; goto _again;} }} + break; + case 138: +#line 695 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_RUNIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 139: +#line 696 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 140: +#line 697 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 141: +#line 698 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 142: +#line 699 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SINHALA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 143: +#line 700 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 144: +#line 701 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 145: +#line 702 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 146: +#line 703 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {cs = stack[--top]; goto _again;} }} + break; + case 147: +#line 704 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 148: +#line 705 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 149: +#line 706 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 150: +#line 707 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {cs = stack[--top]; goto _again;} }} + break; + case 151: +#line 708 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAMIL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 152: +#line 709 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TELUGU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 153: +#line 710 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_THAANA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 154: +#line 711 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_THAI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 155: +#line 712 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 156: +#line 713 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {cs = stack[--top]; goto _again;} }} + break; + case 157: +#line 714 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 158: +#line 715 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_VAI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 159: +#line 716 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_YI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 160: +#line 717 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ANY, negated); {cs = stack[--top]; goto _again;} }} + break; + case 161: +#line 718 "Parser.rl" + {te = p+1;{ throw LocatedParseError("Unknown property"); }} + break; + case 162: +#line 580 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_C, negated); {cs = stack[--top]; goto _again;} }} + break; + case 163: +#line 584 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_CO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 164: +#line 586 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_L, negated); {cs = stack[--top]; goto _again;} }} + break; + case 165: +#line 593 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_M, negated); {cs = stack[--top]; goto _again;} }} + break; + case 166: +#line 595 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_ME, negated); {cs = stack[--top]; goto _again;} }} + break; + case 167: +#line 597 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_N, negated); {cs = stack[--top]; goto _again;} }} + break; + case 168: +#line 601 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_P, negated); {cs = stack[--top]; goto _again;} }} + break; + case 169: +#line 609 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_S, negated); {cs = stack[--top]; goto _again;} }} + break; + case 170: +#line 614 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_Z, negated); {cs = stack[--top]; goto _again;} }} + break; + case 171: +#line 653 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_SCRIPT_HAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 172: +#line 718 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Unknown property"); }} + break; + case 173: +#line 580 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_C, negated); {cs = stack[--top]; goto _again;} }} + break; + case 174: +#line 584 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_CO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 175: +#line 586 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_L, negated); {cs = stack[--top]; goto _again;} }} + break; + case 176: +#line 593 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_M, negated); {cs = stack[--top]; goto _again;} }} + break; + case 177: +#line 595 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_ME, negated); {cs = stack[--top]; goto _again;} }} + break; + case 178: +#line 597 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_N, negated); {cs = stack[--top]; goto _again;} }} + break; + case 179: +#line 601 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_P, negated); {cs = stack[--top]; goto _again;} }} + break; + case 180: +#line 609 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_S, negated); {cs = stack[--top]; goto _again;} }} + break; + case 181: +#line 653 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_SCRIPT_HAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 182: +#line 718 "Parser.rl" + {{p = ((te))-1;}{ throw LocatedParseError("Unknown property"); }} + break; + case 183: +#line 733 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_C, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 184: +#line 741 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_L, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 185: +#line 749 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_M, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 186: +#line 757 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_N, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 187: +#line 765 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_P, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 188: +#line 773 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_S, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 189: +#line 781 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_Z, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 190: +#line 790 "Parser.rl" + {te = p+1;{ throw LocatedParseError("Unknown property"); }} + break; + case 191: +#line 796 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unsupported POSIX collating " + "element"); + }} + break; + case 192: +#line 803 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALNUM, false); + }} + break; + case 193: +#line 806 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALNUM, true); + }} + break; + case 194: +#line 809 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALPHA, false); + }} + break; + case 195: +#line 812 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALPHA, true); + }} + break; + case 196: +#line 815 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ASCII, false); + }} + break; + case 197: +#line 818 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ASCII, true); + }} + break; + case 198: +#line 821 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_BLANK, false); + }} + break; + case 199: +#line 824 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_BLANK, true); + }} + break; + case 200: +#line 827 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_CNTRL, false); + }} + break; + case 201: +#line 830 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_CNTRL, true); + }} + break; + case 202: +#line 833 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, false); + }} + break; + case 203: +#line 836 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, true); + }} + break; + case 204: +#line 839 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_GRAPH, false); + }} + break; + case 205: +#line 842 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_GRAPH, true); + }} + break; + case 206: +#line 845 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_LOWER, false); + }} + break; + case 207: +#line 848 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_LOWER, true); + }} + break; + case 208: +#line 851 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PRINT, false); + }} + break; + case 209: +#line 854 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PRINT, true); + }} + break; + case 210: +#line 857 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PUNCT, false); + }} + break; + case 211: +#line 860 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PUNCT, true); + }} + break; + case 212: +#line 864 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, false); + }} + break; + case 213: +#line 867 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, true); + }} + break; + case 214: +#line 870 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UPPER, false); + }} + break; + case 215: +#line 873 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UPPER, true); + }} + break; + case 216: +#line 876 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, false); + }} + break; + case 217: +#line 879 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, true); + }} + break; + case 218: +#line 882 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_XDIGIT, false); + }} + break; + case 219: +#line 885 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_XDIGIT, true); + }} + break; + case 220: +#line 890 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Invalid POSIX named class"); + }} + break; + case 221: +#line 893 "Parser.rl" + {te = p+1;{ + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 843;goto _again;}} + }} + break; + case 222: +#line 896 "Parser.rl" + {te = p+1;{ /*noop*/}} + break; + case 223: +#line 898 "Parser.rl" + {te = p+1;{ + currentCls->add('\x08'); + }} + break; + case 224: +#line 902 "Parser.rl" + {te = p+1;{ + currentCls->add('\x09'); + }} + break; + case 225: +#line 906 "Parser.rl" + {te = p+1;{ + currentCls->add('\x0a'); + }} + break; + case 226: +#line 910 "Parser.rl" + {te = p+1;{ + currentCls->add('\x0d'); + }} + break; + case 227: +#line 914 "Parser.rl" + {te = p+1;{ + currentCls->add('\x0c'); + }} + break; + case 228: +#line 918 "Parser.rl" + {te = p+1;{ + currentCls->add('\x07'); + }} + break; + case 229: +#line 922 "Parser.rl" + {te = p+1;{ + currentCls->add('\x1b'); + }} + break; + case 230: +#line 926 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_HORZ, false); + }} + break; + case 231: +#line 930 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_HORZ, true); + }} + break; + case 232: +#line 934 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_VERT, false); + }} + break; + case 233: +#line 938 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_VERT, true); + }} + break; + case 234: +#line 942 "Parser.rl" + {te = p+1;{ + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 235: +#line 948 "Parser.rl" + {te = p+1;{ + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 236: +#line 954 "Parser.rl" + {te = p+1;{ + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 237: +#line 960 "Parser.rl" + {te = p+1;{ + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 238: +#line 970 "Parser.rl" + {te = p+1;{ + currentCls->add(octAccumulator); + }} + break; + case 239: +#line 973 "Parser.rl" + {te = p+1;{ + currentCls->add(octAccumulator); + }} + break; + case 240: +#line 977 "Parser.rl" + {te = p+1;{ + string oct(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(oct, nullptr, 8); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if ((!mode.utf8 && val > 255) || val > MAX_UNICODE) { + throw LocatedParseError("Value in \\o{...} sequence is too large"); + } + currentCls->add((unichar)val); + }} + break; + case 241: +#line 997 "Parser.rl" + {te = p+1;{ + currentCls->add(accumulator); + }} + break; + case 242: +#line 1001 "Parser.rl" + {te = p+1;{ + // whatever we found here + currentCls->add(*(ts + 1)); + + }} + break; + case 243: +#line 1007 "Parser.rl" + {te = p+1;{ + string hex(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(hex, nullptr, 16); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if (val > MAX_UNICODE) { + throw LocatedParseError("Value in \\x{...} sequence is too large"); + } + currentCls->add((unichar)val); + }} + break; + case 244: +#line 1025 "Parser.rl" + {te = p+1;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + currentCls->add(decodeCtrl(ts[2])); + } + }} + break; + case 245: +#line 1035 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, false); + }} + break; + case 246: +#line 1039 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, true); + }} + break; + case 247: +#line 1043 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, false); + }} + break; + case 248: +#line 1047 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, true); + }} + break; + case 249: +#line 1051 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, false); + }} + break; + case 250: +#line 1055 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, true); + }} + break; + case 251: +#line 1058 "Parser.rl" + {te = p+1;{ + currentCls->addDash(); + }} + break; + case 252: +#line 276 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported in a character class."; + throw ParseError(str.str()); + }} + break; + case 253: +#line 276 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported in a character class."; + throw ParseError(str.str()); + }} + break; + case 254: +#line 276 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported in a character class."; + throw ParseError(str.str()); + }} + break; + case 255: +#line 1075 "Parser.rl" + {te = p+1;{ + // add the literal char + currentCls->add(*(ts + 1)); + }} + break; + case 256: +#line 1081 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint2c(ts)); + }} + break; + case 257: +#line 1086 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint3c(ts)); + }} + break; + case 258: +#line 1091 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint4c(ts)); + }} + break; + case 259: +#line 1096 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 260: +#line 1102 "Parser.rl" + {te = p+1;{ + currentCls->add((u8)*ts); + }} + break; + case 261: +#line 1106 "Parser.rl" + {te = p+1;{ + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + inCharClass = false; + {cs = 746;goto _again;} + }} + break; + case 262: +#line 966 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 263: +#line 967 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 264: +#line 970 "Parser.rl" + {te = p;p--;{ + currentCls->add(octAccumulator); + }} + break; + case 265: +#line 973 "Parser.rl" + {te = p;p--;{ + currentCls->add(octAccumulator); + }} + break; + case 266: +#line 992 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 267: +#line 997 "Parser.rl" + {te = p;p--;{ + currentCls->add(accumulator); + }} + break; + case 268: +#line 1021 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 269: +#line 1025 "Parser.rl" + {te = p;p--;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + currentCls->add(decodeCtrl(ts[2])); + } + }} + break; + case 270: +#line 1096 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 271: +#line 1102 "Parser.rl" + {te = p;p--;{ + currentCls->add((u8)*ts); + }} + break; + case 272: +#line 992 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 273: +#line 1021 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 274: +#line 1096 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 275: +#line 1102 "Parser.rl" + {{p = ((te))-1;}{ + currentCls->add((u8)*ts); + }} + break; + case 276: +#line 1120 "Parser.rl" + {te = p+1;{ + if (currentCls->isNegated()) { + // Already seen a caret; the second one is not a meta-character. + inCharClassEarly = false; + p--; {cs = 819;goto _again;} + } else { + currentCls->negate(); + // Note: we cannot switch off inCharClassEarly here, as /[^]]/ + // needs to use the right square bracket path below. + } + }} + break; + case 277: +#line 1133 "Parser.rl" + {te = p+1;{ + currentCls->add(']'); + inCharClassEarly = false; + }} + break; + case 278: +#line 1138 "Parser.rl" + {te = p+1;{ { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 843;goto _again;}} }} + break; + case 279: +#line 1139 "Parser.rl" + {te = p+1;{ /*noop*/}} + break; + case 280: +#line 1142 "Parser.rl" + {te = p+1;{ + inCharClassEarly = false; + p--; + {cs = 819;goto _again;} + }} + break; + case 281: +#line 1142 "Parser.rl" + {te = p;p--;{ + inCharClassEarly = false; + p--; + {cs = 819;goto _again;} + }} + break; + case 282: +#line 1154 "Parser.rl" + {te = p+1;{ + {cs = 746;goto _again;} + }} + break; + case 283: +#line 1159 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint2c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 284: +#line 1168 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint3c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 285: +#line 1177 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint4c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 286: +#line 1186 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 287: +#line 1192 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 288: +#line 1186 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 289: +#line 1192 "Parser.rl" + {te = p;p--;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 290: +#line 1186 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 291: +#line 1202 "Parser.rl" + {te = p+1;{ + {cs = stack[--top]; goto _again;} + }} + break; + case 292: +#line 1207 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint2c(ts)); + inCharClassEarly = false; + }} + break; + case 293: +#line 1213 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint3c(ts)); + inCharClassEarly = false; + }} + break; + case 294: +#line 1219 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint4c(ts)); + inCharClassEarly = false; + }} + break; + case 295: +#line 1225 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 296: +#line 1231 "Parser.rl" + {te = p+1;{ + currentCls->add(*ts); + inCharClassEarly = false; + }} + break; + case 297: +#line 1225 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 298: +#line 1231 "Parser.rl" + {te = p;p--;{ + currentCls->add(*ts); + inCharClassEarly = false; + }} + break; + case 299: +#line 1225 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 300: +#line 1243 "Parser.rl" + {te = p+1;{ inComment = false; {cs = 746;goto _again;} }} + break; + case 301: +#line 1247 "Parser.rl" + {te = p+1;} + break; + case 302: +#line 1255 "Parser.rl" + {te = p+1;{ inComment = false; {cs = 746;goto _again;} }} + break; + case 303: +#line 1259 "Parser.rl" + {te = p+1;} + break; + case 304: +#line 1491 "Parser.rl" + {act = 288;} + break; + case 305: +#line 1508 "Parser.rl" + {act = 290;} + break; + case 306: +#line 1737 "Parser.rl" + {act = 330;} + break; + case 307: +#line 362 "Parser.rl" + {te = p+1;{ + if (sequences.empty()) { + throw LocatedParseError("Unmatched parentheses"); + } + currentSeq->finalize(); + POP_SEQUENCE; + }} + break; + case 308: +#line 1274 "Parser.rl" + {te = p+1;{ + currentSeq->addAlternation(); + }} + break; + case 309: +#line 1279 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("POSIX named classes are only " + "supported inside a class"); + }} + break; + case 310: +#line 1286 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unsupported POSIX collating " + "element"); + }} + break; + case 311: +#line 1293 "Parser.rl" + {te = p+1;{ + {cs = 838;goto _again;} + }} + break; + case 312: +#line 1297 "Parser.rl" + {te = p+1;{ /* noop */ }} + break; + case 313: +#line 1299 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); + }} + break; + case 314: +#line 1303 "Parser.rl" + {te = p+1;{ + if (mode.utf8) { + throw LocatedParseError("\\C is unsupported in UTF8"); + } + currentSeq->addComponent(std::make_unique()); + }} + break; + case 315: +#line 1317 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 316: +#line 1324 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 317: +#line 1338 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 318: +#line 1345 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 319: +#line 1359 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat( + 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 320: +#line 1366 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat( + 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 321: +#line 1383 "Parser.rl" + {te = p+1;{ + if (repeatN > repeatM || repeatM == 0) { + throwInvalidRepeat(); + } else if (!currentSeq->addRepeat( + repeatN, repeatM, + ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 322: +#line 1393 "Parser.rl" + {te = p+1;{ + if (repeatN > repeatM || repeatM == 0) { + throwInvalidRepeat(); + } else if (!currentSeq->addRepeat( + repeatN, repeatM, + ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 323: +#line 322 "Parser.rl" + {te = p+1;{ + inComment = true; + {cs = 849;goto _again;} + }} + break; + case 324: +#line 1410 "Parser.rl" + {te = p+1;{ p--; { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 787;goto _again;}} }} + break; + case 325: +#line 1414 "Parser.rl" + {te = p+1;{ assert(0); {p++; goto _out; } }} + break; + case 326: +#line 1421 "Parser.rl" + {te = p+1;{ + auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE + : ComponentBoundary::BEGIN_STRING; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 327: +#line 1428 "Parser.rl" + {te = p+1;{ + auto bound = mode.multiline ? ComponentBoundary::END_LINE + : ComponentBoundary::END_STRING_OPTIONAL_LF; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 328: +#line 1434 "Parser.rl" + {te = p+1;{ + auto bound = ComponentBoundary::BEGIN_STRING; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 329: +#line 1439 "Parser.rl" + {te = p+1;{ + auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 330: +#line 1444 "Parser.rl" + {te = p+1;{ + auto bound = ComponentBoundary::END_STRING; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 331: +#line 1449 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent( + std::make_unique(ts - ptr, false, mode)); + }} + break; + case 332: +#line 1454 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent( + std::make_unique(ts - ptr, true, mode)); + }} + break; + case 333: +#line 1464 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x09', mode); + }} + break; + case 334: +#line 1468 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x0a', mode); + }} + break; + case 335: +#line 1472 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x0d', mode); + }} + break; + case 336: +#line 1476 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x0c', mode); + }} + break; + case 337: +#line 1480 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x07', mode); + }} + break; + case 338: +#line 1484 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x1b', mode); + }} + break; + case 339: +#line 1488 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, octAccumulator, mode); + }} + break; + case 340: +#line 479 "Parser.rl" + {te = p+1;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 341: +#line 486 "Parser.rl" + {te = p+1;{ + // Accumulator is a negative offset. + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + if (accumulator >= groupIndex) { + throw LocatedParseError("Invalid reference"); + } + unsigned idx = groupIndex - accumulator; + currentSeq->addComponent(std::make_unique(idx)); + }} + break; + case 342: +#line 479 "Parser.rl" + {te = p+1;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 343: +#line 486 "Parser.rl" + {te = p+1;{ + // Accumulator is a negative offset. + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + if (accumulator >= groupIndex) { + throw LocatedParseError("Invalid reference"); + } + unsigned idx = groupIndex - accumulator; + currentSeq->addComponent(std::make_unique(idx)); + }} + break; + case 344: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 345: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 346: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 347: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 348: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 349: +#line 1549 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "Onigiruma subroutine call at index " << ts - ptr << + " not supported."; + throw ParseError(str.str()); + }} + break; + case 350: +#line 1560 "Parser.rl" + {te = p+1;{ + string oct(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(oct, nullptr, 8); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if ((!mode.utf8 && val > 255) || val > MAX_UNICODE) { + throw LocatedParseError("Value in \\o{...} sequence is too large"); + } + addEscapedOctal(currentSeq, (unichar)val, mode); + }} + break; + case 351: +#line 1578 "Parser.rl" + {te = p+1;{ + addEscapedHex(currentSeq, accumulator, mode); + }} + break; + case 352: +#line 1582 "Parser.rl" + {te = p+1;{ + string hex(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(hex, nullptr, 16); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if (val > MAX_UNICODE) { + throw LocatedParseError("Value in \\x{...} sequence is too large"); + } + addEscapedHex(currentSeq, (unichar)val, mode); + }} + break; + case 353: +#line 1600 "Parser.rl" + {te = p+1;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + addLiteral(currentSeq, decodeCtrl(ts[2]), mode); + } + }} + break; + case 354: +#line 1610 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported."; + throw ParseError(str.str()); + }} + break; + case 355: +#line 1618 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_WORD, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 356: +#line 1623 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_WORD, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 357: +#line 1628 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_SPACE, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 358: +#line 1633 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_SPACE, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 359: +#line 1638 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_DIGIT, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 360: +#line 1643 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_DIGIT, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 361: +#line 1648 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_HORZ, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 362: +#line 1653 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_HORZ, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 363: +#line 1658 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_VERT, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 364: +#line 1663 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_VERT, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 365: +#line 1668 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 366: +#line 1676 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 367: +#line 1684 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 368: +#line 1692 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 369: +#line 1704 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "\\R at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 370: +#line 1711 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "\\K at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 371: +#line 1726 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "\\G at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 372: +#line 1732 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(ts - ptr, mode)); + }} + break; + case 373: +#line 1737 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, *(ts + 1), mode); + }} + break; + case 374: +#line 316 "Parser.rl" + {te = p+1;{ + inComment = true; + {cs = 848;goto _again;} + }} + break; + case 375: +#line 433 "Parser.rl" + {te = p+1;{ + mode = newMode; + currentSeq->addComponent(std::make_unique()); + }} + break; + case 376: +#line 355 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + mode = newMode; + currentSeq = + enterSequence(currentSeq, std::make_unique()); + }} + break; + case 377: +#line 369 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKAHEAD, + ComponentAssertion::POS)); + }} + break; + case 378: +#line 375 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKAHEAD, + ComponentAssertion::NEG)); + }} + break; + case 379: +#line 381 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKBEHIND, + ComponentAssertion::POS)); + }} + break; + case 380: +#line 387 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKBEHIND, + ComponentAssertion::NEG)); + }} + break; + case 381: +#line 393 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Embedded code is not supported"); + }} + break; + case 382: +#line 393 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Embedded code is not supported"); + }} + break; + case 383: +#line 416 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique()); + }} + break; + case 384: +#line 336 "Parser.rl" + {te = p+1;{ + assert(!label.empty()); // should be guaranteed by machine + char c = *label.begin(); + if (c >= '0' && c <= '9') { + throw LocatedParseError("Group name cannot begin with a digit"); + } + if (!groupNames.insert(label).second) { + throw LocatedParseError("Two named subpatterns use the name '" + label + "'"); + } + PUSH_SEQUENCE; + auto seq = std::make_unique(); + seq->setCaptureIndex(groupIndex++); + seq->setCaptureName(label); + currentSeq = enterSequence(currentSeq, move(seq)); + }} + break; + case 385: +#line 399 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Subpattern reference unsupported"); + }} + break; + case 386: +#line 399 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Subpattern reference unsupported"); + }} + break; + case 387: +#line 1783 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 388: +#line 1794 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 389: +#line 1805 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 390: +#line 1816 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 391: +#line 1828 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Pattern recursion not supported"); + }} + break; + case 392: +#line 402 "Parser.rl" + {te = p+1;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(accumulator)); + }} + break; + case 393: +#line 410 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + assert(!label.empty()); + currentSeq = enterSequence(currentSeq, + std::make_unique(label)); + }} + break; + case 394: +#line 1844 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "Callout at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 395: +#line 1852 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unrecognised character after (?"); + }} + break; + case 396: +#line 1857 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint2c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 397: +#line 1866 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint3c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 398: +#line 1875 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint4c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 399: +#line 1884 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 400: +#line 1893 "Parser.rl" + {te = p+1;{ + if (mode.ignore_space == false) { + addLiteral(currentSeq, *ts, mode); + } + }} + break; + case 401: +#line 1898 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 402: +#line 328 "Parser.rl" + {te = p;p--;{ + PUSH_SEQUENCE; + auto seq = std::make_unique(); + seq->setCaptureIndex(groupIndex++); + currentSeq = enterSequence(currentSeq, move(seq)); + }} + break; + case 403: +#line 421 "Parser.rl" + {te = p;p--;{ + assert(!currentCls); + assert(!inCharClass); // not reentrant + currentCls = getComponentClass(mode); + inCharClass = true; + inCharClassEarly = true; + currentClsBegin = ts; + {cs = 836;goto _again;} + }} + break; + case 404: +#line 1310 "Parser.rl" + {te = p;p--;{ + if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 405: +#line 1331 "Parser.rl" + {te = p;p--;{ + if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 406: +#line 1352 "Parser.rl" + {te = p;p--;{ + if (!currentSeq->addRepeat( + 0, 1, ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 407: +#line 1373 "Parser.rl" + {te = p;p--;{ + if (repeatN > repeatM || repeatM == 0) { + throwInvalidRepeat(); + } else if (!currentSeq->addRepeat( + repeatN, repeatM, + ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 408: +#line 1488 "Parser.rl" + {te = p;p--;{ + addLiteral(currentSeq, octAccumulator, mode); + }} + break; + case 409: +#line 1491 "Parser.rl" + {te = p;p--;{ + // If there are enough capturing sub expressions, this may be + // a back reference + accumulator = parseAsDecimal(octAccumulator); + if (accumulator < groupIndex) { + currentSeq->addComponent(std::make_unique(accumulator)); + } else { + addEscapedOctal(currentSeq, octAccumulator, mode); + } + }} + break; + case 410: +#line 479 "Parser.rl" + {te = p;p--;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 411: +#line 479 "Parser.rl" + {te = p;p--;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 412: +#line 486 "Parser.rl" + {te = p;p--;{ + // Accumulator is a negative offset. + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + if (accumulator >= groupIndex) { + throw LocatedParseError("Invalid reference"); + } + unsigned idx = groupIndex - accumulator; + currentSeq->addComponent(std::make_unique(idx)); + }} + break; + case 413: +#line 1557 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Invalid reference after \\g"); + }} + break; + case 414: +#line 1574 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 415: +#line 1578 "Parser.rl" + {te = p;p--;{ + addEscapedHex(currentSeq, accumulator, mode); + }} + break; + case 416: +#line 1596 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 417: +#line 1600 "Parser.rl" + {te = p;p--;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + addLiteral(currentSeq, decodeCtrl(ts[2]), mode); + } + }} + break; + case 418: +#line 1700 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 419: +#line 1701 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 420: +#line 1719 "Parser.rl" + {te = p;p--;{ + ostringstream str; + str << "\\k at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 421: +#line 1742 "Parser.rl" + {te = p;p--;{ + assert(ts + 1 == pe); + ostringstream str; + str << "Unescaped \\ at end of input, index " << ts - ptr << "."; + throw ParseError(str.str()); + }} + break; + case 422: +#line 396 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Conditional subpattern unsupported"); + }} + break; + case 423: +#line 1852 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Unrecognised character after (?"); + }} + break; + case 424: +#line 1884 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 425: +#line 1898 "Parser.rl" + {te = p;p--;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 426: +#line 328 "Parser.rl" + {{p = ((te))-1;}{ + PUSH_SEQUENCE; + auto seq = std::make_unique(); + seq->setCaptureIndex(groupIndex++); + currentSeq = enterSequence(currentSeq, move(seq)); + }} + break; + case 427: +#line 421 "Parser.rl" + {{p = ((te))-1;}{ + assert(!currentCls); + assert(!inCharClass); // not reentrant + currentCls = getComponentClass(mode); + inCharClass = true; + inCharClassEarly = true; + currentClsBegin = ts; + {cs = 836;goto _again;} + }} + break; + case 428: +#line 1557 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Invalid reference after \\g"); + }} + break; + case 429: +#line 1574 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 430: +#line 1596 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 431: +#line 1719 "Parser.rl" + {{p = ((te))-1;}{ + ostringstream str; + str << "\\k at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 432: +#line 396 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Conditional subpattern unsupported"); + }} + break; + case 433: +#line 1852 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Unrecognised character after (?"); + }} + break; + case 434: +#line 1884 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 435: +#line 1898 "Parser.rl" + {{p = ((te))-1;}{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 436: +#line 1 "NONE" + { switch( act ) { + case 288: + {{p = ((te))-1;} + // If there are enough capturing sub expressions, this may be + // a back reference + accumulator = parseAsDecimal(octAccumulator); + if (accumulator < groupIndex) { + currentSeq->addComponent(std::make_unique(accumulator)); + } else { + addEscapedOctal(currentSeq, octAccumulator, mode); + } + } + break; + case 290: + {{p = ((te))-1;} + // if there are enough left parens to this point, back ref + if (accumulator < groupIndex) { + currentSeq->addComponent(std::make_unique(accumulator)); + } else { + // Otherwise, we interpret the first three digits as an + // octal escape, and the remaining characters stand for + // themselves as literals. + const char *s = ts; + unsigned int accum = 0; + unsigned int oct_digits = 0; + assert(*s == '\\'); // token starts at backslash + for (++s; s < te && oct_digits < 3; ++oct_digits, ++s) { + u8 digit = *s - '0'; + if (digit < 8) { + accum = digit + accum * 8; + } else { + break; + } + } + + if (oct_digits > 0) { + addEscapedOctal(currentSeq, accum, mode); + } + + // And then the rest of the digits, if any, are literal. + for (; s < te; ++s) { + addLiteral(currentSeq, *s, mode); + } + } + } + break; + case 330: + {{p = ((te))-1;} + addLiteral(currentSeq, *(ts + 1), mode); + } + break; + } + } + break; +#line 5508 "Parser.cpp" + } + } + +_again: + _acts = _regex_actions + _regex_to_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 23: +#line 1 "NONE" + {ts = 0;} + break; +#line 5521 "Parser.cpp" + } + } + + if ( cs == 0 ) + goto _out; + if ( ++p != pe ) + goto _resume; + _test_eof: {} + if ( p == eof ) + { + if ( _regex_eof_trans[cs] > 0 ) { + _trans = _regex_eof_trans[cs] - 1; + goto _eof_trans; + } + const short *__acts = _regex_actions + _regex_eof_actions[cs]; + unsigned int __nacts = (unsigned int) *__acts++; + while ( __nacts-- > 0 ) { + switch ( *__acts++ ) { + case 22: +#line 730 "Parser.rl" + { throw LocatedParseError("Malformed property"); } + break; +#line 5544 "Parser.cpp" + } + } + } + + _out: {} + } + +#line 1984 "Parser.rl" + + if (p != pe && *p != '\0') { + // didn't make it to the end of our input, but we didn't throw a ParseError? + assert(0); + ostringstream str; + str << "Parse error at index " << (p - ptr) << "."; + throw ParseError(str.str()); + } + + if (currentCls) { + assert(inCharClass); + assert(currentClsBegin); + ostringstream oss; + oss << "Unterminated character class starting at index " + << currentClsBegin - ptr << "."; + throw ParseError(oss.str()); + } + + if (inComment) { + throw ParseError("Unterminated comment."); + } + + if (!sequences.empty()) { + ostringstream str; + str << "Missing close parenthesis for group started at index " + << sequences.back().seqOffset << "."; + throw ParseError(str.str()); + } + + // Unlikely, but possible + if (groupIndex > 65535) { + throw ParseError("The maximum number of capturing subexpressions is 65535."); + } + + // Finalize the top-level sequence, which will take care of any + // top-level alternation. + currentSeq->finalize(); + assert(currentSeq == rootSeq.get()); + + // Ensure that all references are valid. + checkReferences(*rootSeq, groupIndex, groupNames); + + return move(rootSeq); + } catch (LocatedParseError &error) { + if (ts >= ptr && ts <= pe) { + error.locate(ts - ptr); + } else { + error.locate(0); + } + throw; + } +} + +} // namespace ue2 diff --git a/contrib/vectorscan-cmake/rageled_files/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/control_verbs.cpp new file mode 100644 index 00000000000..19b5c6955e1 --- /dev/null +++ b/contrib/vectorscan-cmake/rageled_files/control_verbs.cpp @@ -0,0 +1,443 @@ + +#line 1 "control_verbs.rl" +/* + * Copyright (c) 2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * \file + * \brief Parser for control verbs that can occur at the beginning of a pattern. + */ + +#include "parser/control_verbs.h" + +#include "parser/Parser.h" +#include "parser/parse_error.h" + +#include +#include + +using namespace std; + +namespace ue2 { + +const char *read_control_verbs(const char *ptr, const char *end, size_t start, + ParseMode &mode) { + const char *p = ptr; + const char *pe = end; + const char *eof = pe; + const char *ts, *te; + int cs; + UNUSED int act; + + +#line 59 "control_verbs.cpp" +static const char _ControlVerbs_actions[] = { + 0, 1, 0, 1, 1, 1, 2, 1, + 3, 1, 4, 1, 5, 1, 6, 1, + 7, 1, 8, 1, 9 +}; + +static const unsigned char _ControlVerbs_key_offsets[] = { + 0, 7, 8, 10, 12, 14, 16, 18, + 20, 21, 23, 25, 27, 30, 32, 34, + 36, 38, 40, 42, 44, 46, 48, 50, + 52, 55, 57, 59, 61, 63, 66, 68, + 70, 72, 74, 76, 79, 82, 84, 86, + 88, 90, 92, 94, 96, 98, 100, 102, + 105, 107, 109, 111, 113, 115, 117, 119, + 121, 123, 125, 127, 129, 131, 133, 135, + 137, 139, 141, 143, 146, 148, 149, 151, + 155, 157, 159, 160, 161 +}; + +static const char _ControlVerbs_trans_keys[] = { + 41, 65, 66, 67, 76, 78, 85, 41, + 41, 78, 41, 89, 41, 67, 41, 82, + 41, 76, 41, 70, 41, 41, 83, 41, + 82, 41, 95, 41, 65, 85, 41, 78, + 41, 89, 41, 67, 41, 78, 41, 73, + 41, 67, 41, 79, 41, 68, 41, 69, + 41, 82, 41, 76, 41, 70, 73, 41, + 77, 41, 73, 41, 84, 41, 95, 41, + 77, 82, 41, 65, 41, 84, 41, 67, + 41, 72, 41, 61, 41, 48, 57, 41, + 48, 57, 41, 69, 41, 67, 41, 85, + 41, 82, 41, 83, 41, 73, 41, 79, + 41, 78, 41, 79, 41, 95, 41, 65, + 83, 41, 85, 41, 84, 41, 79, 41, + 95, 41, 80, 41, 79, 41, 83, 41, + 83, 41, 69, 41, 83, 41, 83, 41, + 84, 41, 65, 41, 82, 41, 84, 41, + 95, 41, 79, 41, 80, 41, 84, 41, + 67, 84, 41, 80, 41, 41, 70, 41, + 49, 51, 56, 41, 54, 41, 50, 41, + 40, 42, 0 +}; + +static const char _ControlVerbs_single_lengths[] = { + 7, 1, 2, 2, 2, 2, 2, 2, + 1, 2, 2, 2, 3, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + 3, 2, 2, 2, 2, 3, 2, 2, + 2, 2, 2, 1, 1, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 3, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 3, 2, 1, 2, 4, + 2, 2, 1, 1, 1 +}; + +static const char _ControlVerbs_range_lengths[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 +}; + +static const short _ControlVerbs_index_offsets[] = { + 0, 8, 10, 13, 16, 19, 22, 25, + 28, 30, 33, 36, 39, 43, 46, 49, + 52, 55, 58, 61, 64, 67, 70, 73, + 76, 80, 83, 86, 89, 92, 96, 99, + 102, 105, 108, 111, 114, 117, 120, 123, + 126, 129, 132, 135, 138, 141, 144, 147, + 151, 154, 157, 160, 163, 166, 169, 172, + 175, 178, 181, 184, 187, 190, 193, 196, + 199, 202, 205, 208, 212, 215, 217, 220, + 225, 228, 231, 233, 235 +}; + +static const char _ControlVerbs_indicies[] = { + 0, 2, 3, 4, 5, 6, 7, 1, + 8, 1, 8, 9, 1, 8, 10, 1, + 11, 12, 1, 8, 13, 1, 8, 14, + 1, 8, 15, 1, 11, 1, 8, 16, + 1, 8, 17, 1, 8, 18, 1, 8, + 19, 20, 1, 8, 21, 1, 8, 22, + 1, 8, 12, 1, 8, 23, 1, 8, + 24, 1, 8, 25, 1, 8, 26, 1, + 8, 27, 1, 8, 15, 1, 8, 28, + 1, 11, 14, 1, 8, 15, 29, 1, + 8, 30, 1, 8, 31, 1, 8, 32, + 1, 8, 33, 1, 8, 34, 35, 1, + 8, 36, 1, 8, 37, 1, 8, 38, + 1, 8, 39, 1, 8, 40, 1, 8, + 41, 1, 11, 41, 1, 8, 42, 1, + 8, 43, 1, 8, 44, 1, 8, 45, + 1, 8, 46, 1, 8, 47, 1, 8, + 48, 1, 8, 39, 1, 8, 49, 1, + 8, 50, 1, 8, 51, 52, 1, 8, + 53, 1, 8, 54, 1, 8, 55, 1, + 8, 56, 1, 8, 57, 1, 8, 58, + 1, 8, 59, 1, 8, 60, 1, 8, + 61, 1, 8, 62, 1, 8, 15, 1, + 8, 63, 1, 8, 64, 1, 8, 65, + 1, 8, 66, 1, 8, 67, 1, 8, + 68, 1, 8, 69, 1, 8, 15, 1, + 8, 70, 71, 1, 8, 72, 1, 73, + 1, 8, 74, 1, 75, 76, 77, 78, + 1, 8, 15, 1, 8, 15, 1, 75, + 1, 80, 79, 82, 81, 0 +}; + +static const char _ControlVerbs_trans_targs[] = { + 75, 1, 2, 9, 22, 24, 45, 67, + 75, 3, 4, 75, 5, 6, 7, 8, + 10, 11, 12, 13, 16, 14, 15, 17, + 18, 19, 20, 21, 23, 25, 26, 27, + 28, 29, 30, 37, 31, 32, 33, 34, + 35, 36, 38, 39, 40, 41, 42, 43, + 44, 46, 47, 48, 59, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 60, + 61, 62, 63, 64, 65, 66, 68, 70, + 69, 75, 71, 75, 72, 73, 74, 75, + 76, 75, 0 +}; + +static const char _ControlVerbs_trans_actions[] = { + 19, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 0, 7, 0, 0, 0, 15, + 5, 17, 0 +}; + +static const char _ControlVerbs_to_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0 +}; + +static const char _ControlVerbs_from_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3, 0 +}; + +static const short _ControlVerbs_eof_trans[] = { + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 82 +}; + +static const int ControlVerbs_start = 75; +static const int ControlVerbs_first_final = 75; +static const int ControlVerbs_error = -1; + +static const int ControlVerbs_en_main = 75; + + +#line 249 "control_verbs.cpp" + { + cs = ControlVerbs_start; + ts = 0; + te = 0; + act = 0; + } + +#line 105 "control_verbs.rl" + + + try { + +#line 262 "control_verbs.cpp" + { + int _klen; + unsigned int _trans; + const char *_acts; + unsigned int _nacts; + const char *_keys; + + if ( p == pe ) + goto _test_eof; +_resume: + _acts = _ControlVerbs_actions + _ControlVerbs_from_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 1: +#line 1 "NONE" + {ts = p;} + break; +#line 281 "control_verbs.cpp" + } + } + + _keys = _ControlVerbs_trans_keys + _ControlVerbs_key_offsets[cs]; + _trans = _ControlVerbs_index_offsets[cs]; + + _klen = _ControlVerbs_single_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + _klen - 1; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + ((_upper-_lower) >> 1); + if ( (*p) < *_mid ) + _upper = _mid - 1; + else if ( (*p) > *_mid ) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); + goto _match; + } + } + _keys += _klen; + _trans += _klen; + } + + _klen = _ControlVerbs_range_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( (*p) < _mid[0] ) + _upper = _mid - 2; + else if ( (*p) > _mid[1] ) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys)>>1); + goto _match; + } + } + _trans += _klen; + } + +_match: + _trans = _ControlVerbs_indicies[_trans]; +_eof_trans: + cs = _ControlVerbs_trans_targs[_trans]; + + if ( _ControlVerbs_trans_actions[_trans] == 0 ) + goto _again; + + _acts = _ControlVerbs_actions + _ControlVerbs_trans_actions[_trans]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) + { + switch ( *_acts++ ) + { + case 2: +#line 1 "NONE" + {te = p+1;} + break; + case 3: +#line 76 "control_verbs.rl" + {te = p+1;{ + mode.utf8 = true; + }} + break; + case 4: +#line 80 "control_verbs.rl" + {te = p+1;{ + mode.ucp = true; + }} + break; + case 5: +#line 84 "control_verbs.rl" + {te = p+1;{ + ostringstream str; + str << "Unsupported control verb " << string(ts, te - ts); + throw LocatedParseError(str.str()); + }} + break; + case 6: +#line 90 "control_verbs.rl" + {te = p+1;{ + ostringstream str; + str << "Unknown control verb " << string(ts, te - ts); + throw LocatedParseError(str.str()); + }} + break; + case 7: +#line 97 "control_verbs.rl" + {te = p+1;{ + p--; + {p++; goto _out; } + }} + break; + case 8: +#line 97 "control_verbs.rl" + {te = p;p--;{ + p--; + {p++; goto _out; } + }} + break; + case 9: +#line 97 "control_verbs.rl" + {{p = ((te))-1;}{ + p--; + {p++; goto _out; } + }} + break; +#line 400 "control_verbs.cpp" + } + } + +_again: + _acts = _ControlVerbs_actions + _ControlVerbs_to_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 0: +#line 1 "NONE" + {ts = 0;} + break; +#line 413 "control_verbs.cpp" + } + } + + if ( ++p != pe ) + goto _resume; + _test_eof: {} + if ( p == eof ) + { + if ( _ControlVerbs_eof_trans[cs] > 0 ) { + _trans = _ControlVerbs_eof_trans[cs] - 1; + goto _eof_trans; + } + } + + _out: {} + } + +#line 109 "control_verbs.rl" + } catch (LocatedParseError &error) { + if (ts >= ptr && ts <= pe) { + error.locate(ts - ptr + start); + } else { + error.locate(0); + } + throw; + } + + return p; +} + +} // namespace ue2 diff --git a/contrib/hyperscan-cmake/x86_64/config.h b/contrib/vectorscan-cmake/x86_64/config.h similarity index 73% rename from contrib/hyperscan-cmake/x86_64/config.h rename to contrib/vectorscan-cmake/x86_64/config.h index 4786e3f4e21..eab2f3eb079 100644 --- a/contrib/hyperscan-cmake/x86_64/config.h +++ b/contrib/vectorscan-cmake/x86_64/config.h @@ -15,15 +15,42 @@ /* "Define if building for EM64T" */ #define ARCH_X86_64 +/* "Define if building for ARM32" */ +/* #undef ARCH_ARM32 */ + +/* "Define if building for AARCH64" */ +/* #undef ARCH_AARCH64 */ + +/* "Define if building for PPC64EL" */ +/* #undef ARCH_PPC64EL */ + +/* "Define if cross compiling for AARCH64" */ +/* #undef CROSS_COMPILE_AARCH64 */ + +/* Define if building SVE for AARCH64. */ +/* #undef BUILD_SVE */ + +/* Define if building SVE2 for AARCH64. */ +/* #undef BUILD_SVE2 */ + +/* Define if building SVE2+BITPERM for AARCH64. */ +/* #undef BUILD_SVE2_BITPERM */ + /* internal build, switch on dump support. */ /* #undef DUMP_SUPPORT */ /* Define if building "fat" runtime. */ /* #undef FAT_RUNTIME */ +/* Define if building AVX2 in the fat runtime. */ +/* #undef BUILD_AVX2 */ + /* Define if building AVX-512 in the fat runtime. */ /* #undef BUILD_AVX512 */ +/* Define if building AVX512VBMI in the fat runtime. */ +/* #undef BUILD_AVX512VBMI */ + /* Define to 1 if `backtrace' works. */ #define HAVE_BACKTRACE @@ -45,6 +72,15 @@ /* C compiler has intrin.h */ /* #undef HAVE_C_INTRIN_H */ +/* C compiler has arm_neon.h */ +/* #undef HAVE_C_ARM_NEON_H */ + +/* C compiler has arm_sve.h */ +/* #undef HAVE_C_ARM_SVE_H */ + +/* C compiler has arm_neon.h */ +/* #undef HAVE_C_PPC64EL_ALTIVEC_H */ + /* Define to 1 if you have the declaration of `pthread_setaffinity_np', and to 0 if you don't. */ /* #undef HAVE_DECL_PTHREAD_SETAFFINITY_NP */ @@ -85,7 +121,7 @@ /* #undef HAVE__ALIGNED_MALLOC */ /* Define if compiler has __builtin_constant_p */ -#define HAVE__BUILTIN_CONSTANT_P +/* #undef HAVE__BUILTIN_CONSTANT_P */ /* Optimize, inline critical functions */ #define HS_OPTIMIZE diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 068377e8f8c..282392bd98a 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -67,7 +67,7 @@ RUN arch=${TARGETARCH:-amd64} \ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper -EXPOSE 2181 10181 44444 +EXPOSE 2181 10181 44444 9181 VOLUME /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 86e56e88aa9..939cd941aeb 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -31,7 +31,7 @@ else DO_CHOWN=0 fi -KEEPER_CONFIG="${KEEPER_CONFIG:-/etc/clickhouse-keeper/config.yaml}" +KEEPER_CONFIG="${KEEPER_CONFIG:-/etc/clickhouse-keeper/keeper_config.xml}" if [ -f "$KEEPER_CONFIG" ] && ! $gosu test -f "$KEEPER_CONFIG" -a -r "$KEEPER_CONFIG"; then echo "Configuration file '$KEEPER_CONFIG' isn't readable by user with id '$USER'" diff --git a/docker/packager/packager b/docker/packager/packager index 14147b8e069..7c0f046b76c 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -202,7 +202,7 @@ def parse_env_variables( cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") if is_release_build(build_type, package_type, sanitizer, split_binary): - cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON") + cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON") result.append("WITH_PERFORMANCE=1") if is_cross_arm: cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1") diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 57e4dfdeda1..d6bd458a01b 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:20.04 # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" + RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update \ @@ -67,7 +68,7 @@ RUN python3 -m pip install \ dict2xml \ dicttoxml \ docker \ - docker-compose==1.28.2 \ + docker-compose==1.29.2 \ grpcio \ grpcio-tools \ kafka-python \ diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 0cb25d12a9f..bcaa064fe4f 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -30,8 +30,8 @@ set -e # cleanup for retry run if volume is not recreated # shellcheck disable=SC2046 { - docker kill $(docker ps -aq) || true - docker rm $(docker ps -aq) || true + docker ps -aq | xargs -r docker kill || true + docker ps -aq | xargs -r docker rm || true } echo "Start tests" diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index 463c08c5304..da866aabd9a 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -73,6 +73,9 @@ function download ) & wait + echo "ATTACH DATABASE default ENGINE=Ordinary" > db0/metadata/default.sql + echo "ATTACH DATABASE datasets ENGINE=Ordinary" > db0/metadata/datasets.sql + ls db0/metadata } download diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 5f55bb9fa21..d77978c904b 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -120,6 +120,10 @@ function run_tests() ADDITIONAL_OPTIONS+=('--replicated-database') fi + if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--db-engine=Ordinary') + fi + set +e clickhouse-test -j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \ --skip 00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" \ diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 52bf8a60669..075f588cae3 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -115,6 +115,10 @@ function run_tests() ADDITIONAL_OPTIONS+=("$RUN_BY_HASH_TOTAL") fi + if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--db-engine=Ordinary') + fi + set +e clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \ --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index c73784c4ef1..0b517fb4af8 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -42,6 +42,7 @@ function install_packages() function configure() { # install test configs + export USE_DATABASE_ORDINARY=1 /usr/share/clickhouse-test/config/install.sh # we mount tests folder from repo to /usr/share @@ -215,7 +216,7 @@ start clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \ || (echo -e 'Server failed to start (see application_errors.txt and clickhouse-server.clean.log)\tFAIL' >> /test_output/test_results.tsv \ - && grep -Fa ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt) + && grep -a ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt) [ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL" [ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL" @@ -284,6 +285,11 @@ then rm -rf /var/lib/clickhouse/* + # Make BC check more funny by forcing Ordinary engine for system database + # New version will try to convert it to Atomic on startup + mkdir /var/lib/clickhouse/metadata + echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql + # Install previous release packages install_packages previous_release_package_folder @@ -313,7 +319,7 @@ then start 500 clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \ || (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \ - && grep -Fa ".*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt) + && grep -a ".*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt) clickhouse-client --query="SELECT 'Server version: ', version()" @@ -343,7 +349,7 @@ then -e "UNFINISHED" \ -e "Renaming unexpected part" \ -e "PART_IS_TEMPORARILY_LOCKED" \ - -e "and a merge is impossible: we didn't find smaller parts" \ + -e "and a merge is impossible: we didn't find" \ -e "found in queue and some source parts for it was lost" \ -e "is lost forever." \ /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "" > /test_output/bc_check_error_messages.txt \ diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index fbff6fd5e97..bfc3ed5e39f 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal==2.1 pytz python-dateutil numpy +RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.2 docker==5.0.0 dicttoxml kazoo tzlocal==2.1 pytz python-dateutil numpy ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 20.10.6 diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index fdd193e9c7f..647989e8421 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -86,7 +86,7 @@ def process_test_log(log_path): test_end = True test_results = [ - (test[0], test[1], test[2], "".join(test[3])) for test in test_results + (test[0], test[1], test[2], "".join(test[3]))[:4096] for test in test_results ] return ( diff --git a/docs/changelogs/v21.1.1.5646-prestable.md b/docs/changelogs/v21.1.1.5646-prestable.md index ec8abc8a05b..97b645c45f9 100644 --- a/docs/changelogs/v21.1.1.5646-prestable.md +++ b/docs/changelogs/v21.1.1.5646-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.1.5646-prestable FIXME as compared to v20.12.1.5236-prestable #### Backward Incompatible Change @@ -257,3 +264,200 @@ * NO CL ENTRY: 'Revert "Add metrics for part number in MergeTree in ClickHouse"'. [#18834](https://github.com/ClickHouse/ClickHouse/pull/18834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Fixed typo in metrics.md'. [#18920](https://github.com/ClickHouse/ClickHouse/pull/18920) ([Mark Frost](https://github.com/frostmark)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Minor change in query profiler [#16899](https://github.com/ClickHouse/ClickHouse/pull/16899) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove escaping from toString(std::string) [#17206](https://github.com/ClickHouse/ClickHouse/pull/17206) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix libunwind build for cmake 3.19+ [#17271](https://github.com/ClickHouse/ClickHouse/pull/17271) ([Azat Khuzhin](https://github.com/azat)). +* Fix AST formatting in log messages [#17274](https://github.com/ClickHouse/ClickHouse/pull/17274) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Tiny cleanup [#17275](https://github.com/ClickHouse/ClickHouse/pull/17275) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for [#13990](https://github.com/ClickHouse/ClickHouse/issues/13990) [#17298](https://github.com/ClickHouse/ClickHouse/pull/17298) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed flaky test_storage_s3::test_custom_auth_headers [#17299](https://github.com/ClickHouse/ClickHouse/pull/17299) ([Pavel Kovalenko](https://github.com/Jokser)). +* Minor changes for ODBC storage [#17301](https://github.com/ClickHouse/ClickHouse/pull/17301) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#16309](https://github.com/ClickHouse/ClickHouse/issues/16309) [#17309](https://github.com/ClickHouse/ClickHouse/pull/17309) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix some flaky tests [#17311](https://github.com/ClickHouse/ClickHouse/pull/17311) ([alesapin](https://github.com/alesapin)). +* Remove outdated test [#17361](https://github.com/ClickHouse/ClickHouse/pull/17361) ([Anton Popov](https://github.com/CurtizJ)). +* Added a test for what was always working [#17375](https://github.com/ClickHouse/ClickHouse/pull/17375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for mutation with empty partition [#17410](https://github.com/ClickHouse/ClickHouse/pull/17410) ([alesapin](https://github.com/alesapin)). +* Implement GRPC protocol (corrections) [#17435](https://github.com/ClickHouse/ClickHouse/pull/17435) ([Vitaly Baranov](https://github.com/vitlibar)). +* Drop include of the removed libbtrie in cmake rules [#17454](https://github.com/ClickHouse/ClickHouse/pull/17454) ([Azat Khuzhin](https://github.com/azat)). +* Merge expressions [#17458](https://github.com/ClickHouse/ClickHouse/pull/17458) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fail queries on unknown settings in perf tests [#17460](https://github.com/ClickHouse/ClickHouse/pull/17460) ([Azat Khuzhin](https://github.com/azat)). +* Adjust perf test thresholds [#17485](https://github.com/ClickHouse/ClickHouse/pull/17485) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix more flaky integration tests [#17486](https://github.com/ClickHouse/ClickHouse/pull/17486) ([alesapin](https://github.com/alesapin)). +* Fix data race on global BlockStreamProfileInfo in PullingAsyncPipelineExecutor [#17498](https://github.com/ClickHouse/ClickHouse/pull/17498) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix CMake generation and build for native Xcode and AppleClang [#17501](https://github.com/ClickHouse/ClickHouse/pull/17501) ([Denis Glazachev](https://github.com/traceon)). +* Fix bad test 01317_no_password_in_command_line.sh [#17506](https://github.com/ClickHouse/ClickHouse/pull/17506) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix spelling errors [#17527](https://github.com/ClickHouse/ClickHouse/pull/17527) ([flynn](https://github.com/ucasfl)). +* Fix toUnixTimestamp(Date()) error (use type name not column type name) [#17536](https://github.com/ClickHouse/ClickHouse/pull/17536) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for StorageJoin and UUID [#17541](https://github.com/ClickHouse/ClickHouse/pull/17541) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra for BoringSSL [#17544](https://github.com/ClickHouse/ClickHouse/pull/17544) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Kafka for BoringSSL [#17545](https://github.com/ClickHouse/ClickHouse/pull/17545) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update MariaDB for BoringSSL [#17546](https://github.com/ClickHouse/ClickHouse/pull/17546) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update OpenLDAP for BoringSSL [#17547](https://github.com/ClickHouse/ClickHouse/pull/17547) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update AMQP for BoringSSL [#17548](https://github.com/ClickHouse/ClickHouse/pull/17548) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* getMemoryAmount: make code worse [#17556](https://github.com/ClickHouse/ClickHouse/pull/17556) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#12297](https://github.com/ClickHouse/ClickHouse/issues/12297) [#17557](https://github.com/ClickHouse/ClickHouse/pull/17557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9490](https://github.com/ClickHouse/ClickHouse/issues/9490) [#17561](https://github.com/ClickHouse/ClickHouse/pull/17561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11803](https://github.com/ClickHouse/ClickHouse/issues/11803) [#17562](https://github.com/ClickHouse/ClickHouse/pull/17562) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix mixed statements in 01018_ip_dictionary.sql [#17570](https://github.com/ClickHouse/ClickHouse/pull/17570) ([Vladimir C](https://github.com/vdimir)). +* Fix GRPC tests [#17597](https://github.com/ClickHouse/ClickHouse/pull/17597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Port ClickHouse code to BoringSSL [#17606](https://github.com/ClickHouse/ClickHouse/pull/17606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the OpenTelemetry test [#17621](https://github.com/ClickHouse/ClickHouse/pull/17621) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Try to fix Yandex Synchronization check [#17634](https://github.com/ClickHouse/ClickHouse/pull/17634) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use feature testing macro to test if char8_t is supported [#17645](https://github.com/ClickHouse/ClickHouse/pull/17645) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Make LockExceptionInThread/BlockerInThread nested [#17658](https://github.com/ClickHouse/ClickHouse/pull/17658) ([Azat Khuzhin](https://github.com/azat)). +* Mark grpc protocol's tests as flaky. [#17662](https://github.com/ClickHouse/ClickHouse/pull/17662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Use feature testing macro once more [#17685](https://github.com/ClickHouse/ClickHouse/pull/17685) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Fix sequential number in TestKeeper [#17700](https://github.com/ClickHouse/ClickHouse/pull/17700) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix arcadia build [#17720](https://github.com/ClickHouse/ClickHouse/pull/17720) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix arcadian build [#17781](https://github.com/ClickHouse/ClickHouse/pull/17781) ([Ivan](https://github.com/abyss7)). +* Fix a typo [#17791](https://github.com/ClickHouse/ClickHouse/pull/17791) ([achimbab](https://github.com/achimbab)). +* Attempt to use IOStream in AWS SDK [#17794](https://github.com/ClickHouse/ClickHouse/pull/17794) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Update StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry to support waiting on foreign shards / tables [#17800](https://github.com/ClickHouse/ClickHouse/pull/17800) ([nvartolomei](https://github.com/nvartolomei)). +* Fix flaky test_ttl_move [#17805](https://github.com/ClickHouse/ClickHouse/pull/17805) ([Azat Khuzhin](https://github.com/azat)). +* Merging data type Map [#15806](https://github.com/ClickHouse/ClickHouse/issues/15806) [#17829](https://github.com/ClickHouse/ClickHouse/pull/17829) ([Anton Popov](https://github.com/CurtizJ)). +* Kill network container with retries in integration tests [#17856](https://github.com/ClickHouse/ClickHouse/pull/17856) ([alesapin](https://github.com/alesapin)). +* Merging [#17750](https://github.com/ClickHouse/ClickHouse/issues/17750) [#17874](https://github.com/ClickHouse/ClickHouse/pull/17874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix bad code [#17878](https://github.com/ClickHouse/ClickHouse/pull/17878) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Retrieve logs from grpc [#17888](https://github.com/ClickHouse/ClickHouse/pull/17888) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed typo in log message format [#17900](https://github.com/ClickHouse/ClickHouse/pull/17900) ([Alexander Kazakov](https://github.com/Akazz)). +* Perf tests fixes [#17907](https://github.com/ClickHouse/ClickHouse/pull/17907) ([Azat Khuzhin](https://github.com/azat)). +* Better exception message for MaterializeMySQL [#17915](https://github.com/ClickHouse/ClickHouse/pull/17915) ([Winter Zhang](https://github.com/zhang2014)). +* Add additional columns size check for MergeTree in debug mode [#17919](https://github.com/ClickHouse/ClickHouse/pull/17919) ([alesapin](https://github.com/alesapin)). +* Small simplification of MergeTreeDataWriter [#17943](https://github.com/ClickHouse/ClickHouse/pull/17943) ([alesapin](https://github.com/alesapin)). +* Use ryu instead of dragonbox in Arcadia [#17963](https://github.com/ClickHouse/ClickHouse/pull/17963) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Tiny build changes [#17982](https://github.com/ClickHouse/ClickHouse/pull/17982) ([Azat Khuzhin](https://github.com/azat)). +* Fix arcadia [#17984](https://github.com/ClickHouse/ClickHouse/pull/17984) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* More forward declaration for generic headers [#17986](https://github.com/ClickHouse/ClickHouse/pull/17986) ([Azat Khuzhin](https://github.com/azat)). +* Remove some redundant includes to speed up build [#17988](https://github.com/ClickHouse/ClickHouse/pull/17988) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flappy test_log_family_s3 [#18027](https://github.com/ClickHouse/ClickHouse/pull/18027) ([Pavel Kovalenko](https://github.com/Jokser)). +* encodeXMLComponent: rename files after [#17659](https://github.com/ClickHouse/ClickHouse/issues/17659) [#18033](https://github.com/ClickHouse/ClickHouse/pull/18033) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* HashTable erase added tests [#18047](https://github.com/ClickHouse/ClickHouse/pull/18047) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove obsolete settings [#18054](https://github.com/ClickHouse/ClickHouse/pull/18054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01600_remerge_sort_lowered_memory_bytes_ratio flap [#18057](https://github.com/ClickHouse/ClickHouse/pull/18057) ([Azat Khuzhin](https://github.com/azat)). +* Remove test_keeper_server in perf tests [#18058](https://github.com/ClickHouse/ClickHouse/pull/18058) ([Azat Khuzhin](https://github.com/azat)). +* Add changelog for 20.12 [#18062](https://github.com/ClickHouse/ClickHouse/pull/18062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed unsafe ast rewrite in InterpreterSelectWithUnionQuery [#18064](https://github.com/ClickHouse/ClickHouse/pull/18064) ([Alexander Kazakov](https://github.com/Akazz)). +* Build utils in CI, at least in split build [#18066](https://github.com/ClickHouse/ClickHouse/pull/18066) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Do not pass -fsanitize-blacklist for gcc (it does not support it) for UBSAN [#18081](https://github.com/ClickHouse/ClickHouse/pull/18081) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix Arcadia [#18084](https://github.com/ClickHouse/ClickHouse/pull/18084) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix dependencies for docker stateful_with_coverage [#18105](https://github.com/ClickHouse/ClickHouse/pull/18105) ([alesapin](https://github.com/alesapin)). +* Avoid using symlinks for top_level_domains [#18113](https://github.com/ClickHouse/ClickHouse/pull/18113) ([Azat Khuzhin](https://github.com/azat)). +* gcc10 sanitizers support [#18114](https://github.com/ClickHouse/ClickHouse/pull/18114) ([Azat Khuzhin](https://github.com/azat)). +* Port Kerberos to BoringSSL [#18128](https://github.com/ClickHouse/ClickHouse/pull/18128) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fix integration tests. [#18132](https://github.com/ClickHouse/ClickHouse/pull/18132) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Trying to fix 00620_optimize_on_nonleader_test [#18140](https://github.com/ClickHouse/ClickHouse/pull/18140) ([alesapin](https://github.com/alesapin)). +* Suppress error in 00993_system_parts_race_condition_drop_zookeeper [#18148](https://github.com/ClickHouse/ClickHouse/pull/18148) ([alesapin](https://github.com/alesapin)). +* Better exception message for unknown function [#18168](https://github.com/ClickHouse/ClickHouse/pull/18168) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Test watches for in-memory zookeeper [#18191](https://github.com/ClickHouse/ClickHouse/pull/18191) ([alesapin](https://github.com/alesapin)). +* Add support for LTS branches in backport automation [#18195](https://github.com/ClickHouse/ClickHouse/pull/18195) ([Ivan](https://github.com/abyss7)). +* Enable optimize_on_insert for MaterializeMySQL [#18198](https://github.com/ClickHouse/ClickHouse/pull/18198) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add one more case in zk test util [#18199](https://github.com/ClickHouse/ClickHouse/pull/18199) ([alesapin](https://github.com/alesapin)). +* More logs during quorum insert [#18200](https://github.com/ClickHouse/ClickHouse/pull/18200) ([alesapin](https://github.com/alesapin)). +* Update libc headers [#18202](https://github.com/ClickHouse/ClickHouse/pull/18202) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix watches processing in TestKeeper [#18217](https://github.com/ClickHouse/ClickHouse/pull/18217) ([alesapin](https://github.com/alesapin)). +* Add test for fixed bug with skip indices [#18219](https://github.com/ClickHouse/ClickHouse/pull/18219) ([Anton Popov](https://github.com/CurtizJ)). +* [wip] a prototype for window functions [#18222](https://github.com/ClickHouse/ClickHouse/pull/18222) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Return clickhouse-git-import [#18234](https://github.com/ClickHouse/ClickHouse/pull/18234) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* MADV_DONTNEED check in runtime for qemu (via patching jemalloc) [#18238](https://github.com/ClickHouse/ClickHouse/pull/18238) ([Azat Khuzhin](https://github.com/azat)). +* New Year preparations [#18274](https://github.com/ClickHouse/ClickHouse/pull/18274) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add powf and powl to glibc-compatibility [#18279](https://github.com/ClickHouse/ClickHouse/pull/18279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#18286](https://github.com/ClickHouse/ClickHouse/pull/18286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 01076_parallel_alter test [#18293](https://github.com/ClickHouse/ClickHouse/pull/18293) ([alesapin](https://github.com/alesapin)). +* Make single image for building coverage report [#18312](https://github.com/ClickHouse/ClickHouse/pull/18312) ([alesapin](https://github.com/alesapin)). +* Fixed flaky test [#18313](https://github.com/ClickHouse/ClickHouse/pull/18313) ([Vasily Nemkov](https://github.com/Enmk)). +* Perf test for ColumnMap [#18317](https://github.com/ClickHouse/ClickHouse/pull/18317) ([Vasily Nemkov](https://github.com/Enmk)). +* Add more tests to skip-list [#18318](https://github.com/ClickHouse/ClickHouse/pull/18318) ([Ivan](https://github.com/abyss7)). +* Add support for ANTLR inside clickhouse-test [#18319](https://github.com/ClickHouse/ClickHouse/pull/18319) ([Ivan](https://github.com/abyss7)). +* Fix clickhouse-test [#18330](https://github.com/ClickHouse/ClickHouse/pull/18330) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Provide extra constructor for Async metrics [#18331](https://github.com/ClickHouse/ClickHouse/pull/18331) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* tests: remove unused configs/scripts [#18334](https://github.com/ClickHouse/ClickHouse/pull/18334) ([Azat Khuzhin](https://github.com/azat)). +* Fix log message for memory tracking drift [#18335](https://github.com/ClickHouse/ClickHouse/pull/18335) ([Azat Khuzhin](https://github.com/azat)). +* try to pass ninja flags in deb package [#18348](https://github.com/ClickHouse/ClickHouse/pull/18348) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove useless code [#18350](https://github.com/ClickHouse/ClickHouse/pull/18350) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused code [#18366](https://github.com/ClickHouse/ClickHouse/pull/18366) ([Anton Popov](https://github.com/CurtizJ)). +* Return back some configs, that are used in Arcadia [#18370](https://github.com/ClickHouse/ClickHouse/pull/18370) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Return clickhouse-test-server for Arcadia needs [#18372](https://github.com/ClickHouse/ClickHouse/pull/18372) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Drop libnsl dependency (because of gRPC -> c-ares) [#18375](https://github.com/ClickHouse/ClickHouse/pull/18375) ([Azat Khuzhin](https://github.com/azat)). +* bump compatibility level to 10 for debian manifests [#18376](https://github.com/ClickHouse/ClickHouse/pull/18376) ([Azat Khuzhin](https://github.com/azat)). +* Do not override RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK in rocksdb [#18378](https://github.com/ClickHouse/ClickHouse/pull/18378) ([Azat Khuzhin](https://github.com/azat)). +* Make some perf tests faster on slower machines [#18386](https://github.com/ClickHouse/ClickHouse/pull/18386) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix exception text from Pipe.cpp [#18396](https://github.com/ClickHouse/ClickHouse/pull/18396) ([filimonov](https://github.com/filimonov)). +* Add test for already working code [#18405](https://github.com/ClickHouse/ClickHouse/pull/18405) ([alesapin](https://github.com/alesapin)). +* Try fix ya.make [#18409](https://github.com/ClickHouse/ClickHouse/pull/18409) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use Port::Data instead of Chunk in LazyOutputFormat. [#18411](https://github.com/ClickHouse/ClickHouse/pull/18411) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not use watchdog when server is run from tty [#18433](https://github.com/ClickHouse/ClickHouse/pull/18433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Raise an error if more than one key is provided to ip_dictionary [#18435](https://github.com/ClickHouse/ClickHouse/pull/18435) ([Vladimir C](https://github.com/vdimir)). +* Remove unused code [#18436](https://github.com/ClickHouse/ClickHouse/pull/18436) ([Anton Popov](https://github.com/CurtizJ)). +* Ignore SOURCE_DATE_EPOCH for newer ccache (4+) [#18441](https://github.com/ClickHouse/ClickHouse/pull/18441) ([Azat Khuzhin](https://github.com/azat)). +* Poco build fixes [#18443](https://github.com/ClickHouse/ClickHouse/pull/18443) ([Azat Khuzhin](https://github.com/azat)). +* [wip] some window function fixes [#18455](https://github.com/ClickHouse/ClickHouse/pull/18455) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Merging [#17858](https://github.com/ClickHouse/ClickHouse/issues/17858) [#18475](https://github.com/ClickHouse/ClickHouse/pull/18475) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow AppleClang builds [#18488](https://github.com/ClickHouse/ClickHouse/pull/18488) ([Denis Glazachev](https://github.com/traceon)). +* Build job pool tiny fixes [#18489](https://github.com/ClickHouse/ClickHouse/pull/18489) ([Azat Khuzhin](https://github.com/azat)). +* Add NuRaft to contrib [#18491](https://github.com/ClickHouse/ClickHouse/pull/18491) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 01584_distributed_buffer_cannot_find_column [#18493](https://github.com/ClickHouse/ClickHouse/pull/18493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More correct error code on incorrect interserver protocol [#18515](https://github.com/ClickHouse/ClickHouse/pull/18515) ([alesapin](https://github.com/alesapin)). +* Merging [#18188](https://github.com/ClickHouse/ClickHouse/issues/18188) [#18516](https://github.com/ClickHouse/ClickHouse/pull/18516) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix usage of concurrent bounded queue in test keeper storage [#18522](https://github.com/ClickHouse/ClickHouse/pull/18522) ([alesapin](https://github.com/alesapin)). +* Fix cast to map from tuple of arrays with unequal sizes [#18523](https://github.com/ClickHouse/ClickHouse/pull/18523) ([Anton Popov](https://github.com/CurtizJ)). +* Sim/Min Hash fixes [#18524](https://github.com/ClickHouse/ClickHouse/pull/18524) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use RLIMIT_DATA/RLIMIT_AS over RLIMIT_RSS for ENABLE_CHECK_HEAVY_BUILDS [#18537](https://github.com/ClickHouse/ClickHouse/pull/18537) ([Azat Khuzhin](https://github.com/azat)). +* Do not throw logical error from IPAddressDictionary ctor [#18548](https://github.com/ClickHouse/ClickHouse/pull/18548) ([Vladimir C](https://github.com/vdimir)). +* Check for CLICKHOUSE_CLIENT_OPT env before setting it [#18574](https://github.com/ClickHouse/ClickHouse/pull/18574) ([Ivan](https://github.com/abyss7)). +* Minor fixes for min/sim hash [#18595](https://github.com/ClickHouse/ClickHouse/pull/18595) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change union_default_mode to throw exception [#18615](https://github.com/ClickHouse/ClickHouse/pull/18615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed exit code of watchdog [#18616](https://github.com/ClickHouse/ClickHouse/pull/18616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small utils improvements (check-marks and compressor) [#18619](https://github.com/ClickHouse/ClickHouse/pull/18619) ([Azat Khuzhin](https://github.com/azat)). +* Fix too long perf test [#18634](https://github.com/ClickHouse/ClickHouse/pull/18634) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More correct words about parser [#18646](https://github.com/ClickHouse/ClickHouse/pull/18646) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless support for symbolic port names [#18647](https://github.com/ClickHouse/ClickHouse/pull/18647) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Send fatal logs in all tests [#18648](https://github.com/ClickHouse/ClickHouse/pull/18648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix AppleClang compilation - Remove auto in function parameters [#18674](https://github.com/ClickHouse/ClickHouse/pull/18674) ([Denis Glazachev](https://github.com/traceon)). +* compressor: remove extra check for seeking of input [#18675](https://github.com/ClickHouse/ClickHouse/pull/18675) ([Azat Khuzhin](https://github.com/azat)). +* Better linker name matcher [#18678](https://github.com/ClickHouse/ClickHouse/pull/18678) ([Amos Bird](https://github.com/amosbird)). +* Remove "harmful" function from mariadbclient [#18682](https://github.com/ClickHouse/ClickHouse/pull/18682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance comparison [#18686](https://github.com/ClickHouse/ClickHouse/pull/18686) ([Azat Khuzhin](https://github.com/azat)). +* Simplify code of function "bar" [#18687](https://github.com/ClickHouse/ClickHouse/pull/18687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge byteSize function [#18688](https://github.com/ClickHouse/ClickHouse/pull/18688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow Replicated tables in Arcadia [#18697](https://github.com/ClickHouse/ClickHouse/pull/18697) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Bump replxx to fix loading of multi-line entries from the history [#18700](https://github.com/ClickHouse/ClickHouse/pull/18700) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for already fixed issue [#18702](https://github.com/ClickHouse/ClickHouse/pull/18702) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report: applying non-zero offset to nullptr [#18703](https://github.com/ClickHouse/ClickHouse/pull/18703) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable more tests and see what will happen [#18704](https://github.com/ClickHouse/ClickHouse/pull/18704) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a test for [#13477](https://github.com/ClickHouse/ClickHouse/issues/13477) [#18708](https://github.com/ClickHouse/ClickHouse/pull/18708) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzz errors in sumMap [#18710](https://github.com/ClickHouse/ClickHouse/pull/18710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use relaxed for flag in RemoteQueryExecutorReadContext. [#18715](https://github.com/ClickHouse/ClickHouse/pull/18715) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not throw from Parser [#18745](https://github.com/ClickHouse/ClickHouse/pull/18745) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove pink screen with confusing questions about Kerberos [#18748](https://github.com/ClickHouse/ClickHouse/pull/18748) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't allow conversion between UUID and numeric types [#18749](https://github.com/ClickHouse/ClickHouse/pull/18749) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove bad code in HashJoin [#18750](https://github.com/ClickHouse/ClickHouse/pull/18750) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More robust stateful test [#18751](https://github.com/ClickHouse/ClickHouse/pull/18751) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test from [#15641](https://github.com/ClickHouse/ClickHouse/issues/15641) [#18753](https://github.com/ClickHouse/ClickHouse/pull/18753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark some TestFlows as flaky [#18757](https://github.com/ClickHouse/ClickHouse/pull/18757) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some headers [#18758](https://github.com/ClickHouse/ClickHouse/pull/18758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a regression test for marks corruption [#18763](https://github.com/ClickHouse/ClickHouse/pull/18763) ([Azat Khuzhin](https://github.com/azat)). +* Use sigdescr_np() over sys_siglist (fixes glibc 2.32+ unbundled build) [#18764](https://github.com/ClickHouse/ClickHouse/pull/18764) ([Azat Khuzhin](https://github.com/azat)). +* Do not materialize block for FetchColumns header. [#18768](https://github.com/ClickHouse/ClickHouse/pull/18768) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Query Fuzzer: fix some cultural issues [#18770](https://github.com/ClickHouse/ClickHouse/pull/18770) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* more tests for simple aggregate functions in summingMT aggregatingMT try2 [#18771](https://github.com/ClickHouse/ClickHouse/pull/18771) ([Denny Crane](https://github.com/den-crane)). +* Check if XCODE_IDE is true and avoid enforcing ninja in that case [#18773](https://github.com/ClickHouse/ClickHouse/pull/18773) ([Denis Glazachev](https://github.com/traceon)). +* Respect memory tracker blocker level during deallocations [#18774](https://github.com/ClickHouse/ClickHouse/pull/18774) ([Azat Khuzhin](https://github.com/azat)). +* Do not allow Fuzzer to enable LLVM [#18777](https://github.com/ClickHouse/ClickHouse/pull/18777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add sanity checks for Sim/Min hash arguments [#18803](https://github.com/ClickHouse/ClickHouse/pull/18803) ([Azat Khuzhin](https://github.com/azat)). +* Respect MINSIGSTKSZ for alternative stack to fix under aarch64 [#18832](https://github.com/ClickHouse/ClickHouse/pull/18832) ([Azat Khuzhin](https://github.com/azat)). +* Do not check bit flips for big buffers (since the size can be corrupted) [#18852](https://github.com/ClickHouse/ClickHouse/pull/18852) ([Azat Khuzhin](https://github.com/azat)). +* Correctly override default settings remotely [#18857](https://github.com/ClickHouse/ClickHouse/pull/18857) ([Amos Bird](https://github.com/amosbird)). +* Import strsignal from Musl [#18858](https://github.com/ClickHouse/ClickHouse/pull/18858) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Storage features improvements [#18870](https://github.com/ClickHouse/ClickHouse/pull/18870) ([Azat Khuzhin](https://github.com/azat)). +* Fix integrity check [#18871](https://github.com/ClickHouse/ClickHouse/pull/18871) ([Azat Khuzhin](https://github.com/azat)). +* Minor fix in backport script [#18873](https://github.com/ClickHouse/ClickHouse/pull/18873) ([Ivan](https://github.com/abyss7)). +* Query Fuzzer: return fail fast semantics [#18880](https://github.com/ClickHouse/ClickHouse/pull/18880) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless headers [#18881](https://github.com/ClickHouse/ClickHouse/pull/18881) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use time_macros over unset SOURCE_DATE_EPOCH in ccache 4.2 (unreleased) [#18885](https://github.com/ClickHouse/ClickHouse/pull/18885) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#18886](https://github.com/ClickHouse/ClickHouse/pull/18886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TestFlows due to timeouts [#18888](https://github.com/ClickHouse/ClickHouse/pull/18888) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless headers [#18892](https://github.com/ClickHouse/ClickHouse/pull/18892) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Drop RESTART REPLICAS from stateless tests to avoid locking lots of mutexes [#18897](https://github.com/ClickHouse/ClickHouse/pull/18897) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.1.2.15-stable.md b/docs/changelogs/v21.1.2.15-stable.md index 205794b94c2..ac76c79ea4b 100644 --- a/docs/changelogs/v21.1.2.15-stable.md +++ b/docs/changelogs/v21.1.2.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.2.15-stable FIXME as compared to v21.1.1.5646-prestable #### Improvement diff --git a/docs/changelogs/v21.1.3.32-stable.md b/docs/changelogs/v21.1.3.32-stable.md index ea4c9fd0fe6..85e7162a509 100644 --- a/docs/changelogs/v21.1.3.32-stable.md +++ b/docs/changelogs/v21.1.3.32-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.3.32-stable FIXME as compared to v21.1.2.15-stable #### Bug Fix @@ -29,3 +36,7 @@ * Backported in [#19938](https://github.com/ClickHouse/ClickHouse/issues/19938): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#19934](https://github.com/ClickHouse/ClickHouse/issues/19934): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.1.4.46-stable.md b/docs/changelogs/v21.1.4.46-stable.md index 3033c5edd21..cc9010880e4 100644 --- a/docs/changelogs/v21.1.4.46-stable.md +++ b/docs/changelogs/v21.1.4.46-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.4.46-stable FIXME as compared to v21.1.3.32-stable #### Bug Fix @@ -20,3 +27,8 @@ * NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.1: Fix access control manager destruction order"'. [#20395](https://github.com/ClickHouse/ClickHouse/pull/20395) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.1.5.4-stable.md b/docs/changelogs/v21.1.5.4-stable.md index c67724a4512..277814b74be 100644 --- a/docs/changelogs/v21.1.5.4-stable.md +++ b/docs/changelogs/v21.1.5.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.5.4-stable FIXME as compared to v21.1.4.46-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.6.13-stable.md b/docs/changelogs/v21.1.6.13-stable.md index 547cd38a06f..5308dc43dea 100644 --- a/docs/changelogs/v21.1.6.13-stable.md +++ b/docs/changelogs/v21.1.6.13-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.6.13-stable FIXME as compared to v21.1.5.4-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.7.1-stable.md b/docs/changelogs/v21.1.7.1-stable.md index 371efb8d5df..0e66e1bc623 100644 --- a/docs/changelogs/v21.1.7.1-stable.md +++ b/docs/changelogs/v21.1.7.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.7.1-stable FIXME as compared to v21.1.6.13-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.8.30-stable.md b/docs/changelogs/v21.1.8.30-stable.md index 0859cc8ccbd..f9776ce4810 100644 --- a/docs/changelogs/v21.1.8.30-stable.md +++ b/docs/changelogs/v21.1.8.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.8.30-stable FIXME as compared to v21.1.7.1-stable #### Bug Fix @@ -27,3 +34,8 @@ * Backported in [#22700](https://github.com/ClickHouse/ClickHouse/issues/22700): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). * Backported in [#22739](https://github.com/ClickHouse/ClickHouse/issues/22739): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* Mannual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.1 [#22506](https://github.com/ClickHouse/ClickHouse/pull/22506) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.1.9.41-stable.md b/docs/changelogs/v21.1.9.41-stable.md index 2a94073b810..bef4385e2cb 100644 --- a/docs/changelogs/v21.1.9.41-stable.md +++ b/docs/changelogs/v21.1.9.41-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.9.41-stable FIXME as compared to v21.1.8.30-stable #### Improvement @@ -15,3 +22,8 @@ #### Build/Testing/Packaging Improvement * Backported in [#22813](https://github.com/ClickHouse/ClickHouse/issues/22813): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.1.8013-prestable.md b/docs/changelogs/v21.10.1.8013-prestable.md index d3e06c056cf..506f3642287 100644 --- a/docs/changelogs/v21.10.1.8013-prestable.md +++ b/docs/changelogs/v21.10.1.8013-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.1.8013-prestable FIXME as compared to v21.9.1.7770-prestable #### Backward Incompatible Change @@ -136,3 +143,126 @@ * NO CL ENTRY: 'Revert "Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398)"'. [#28274](https://github.com/ClickHouse/ClickHouse/pull/28274) ([Alexander Tokmakov](https://github.com/tavplubix)). * NO CL ENTRY: 'fix minor typo'. [#28629](https://github.com/ClickHouse/ClickHouse/pull/28629) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Experiment with asynchronous readers [#26791](https://github.com/ClickHouse/ClickHouse/pull/26791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Introduce sessions [#26864](https://github.com/ClickHouse/ClickHouse/pull/26864) ([Vitaly Baranov](https://github.com/vitlibar)). +* FILTER clause for aggregate functions [#27036](https://github.com/ClickHouse/ClickHouse/pull/27036) ([Nikita Taranov](https://github.com/nickitat)). +* Add test for parsing maps with integer keys [#27146](https://github.com/ClickHouse/ClickHouse/pull/27146) ([Anton Popov](https://github.com/CurtizJ)). +* S3 disk unstable reads test [#27176](https://github.com/ClickHouse/ClickHouse/pull/27176) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Refactor NotJoined [#27299](https://github.com/ClickHouse/ClickHouse/pull/27299) ([Vladimir C](https://github.com/vdimir)). +* Accept error code by error name in client test hints [#27430](https://github.com/ClickHouse/ClickHouse/pull/27430) ([Azat Khuzhin](https://github.com/azat)). +* Break some tests [#27529](https://github.com/ClickHouse/ClickHouse/pull/27529) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove the remains of ANTLR in the tests [#27637](https://github.com/ClickHouse/ClickHouse/pull/27637) ([Raúl Marín](https://github.com/Algunenano)). +* Disable memory tracking for roaring bitmaps on Mac OS [#27681](https://github.com/ClickHouse/ClickHouse/pull/27681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use only SSE2 in "unbundled" build [#27683](https://github.com/ClickHouse/ClickHouse/pull/27683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash [#27685](https://github.com/ClickHouse/ClickHouse/pull/27685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix stress test in `~CompressedWriteBuffer` [#27686](https://github.com/ClickHouse/ClickHouse/pull/27686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark tests for `DatabaseReplicated` as green [#27688](https://github.com/ClickHouse/ClickHouse/pull/27688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed DenseHashMap, DenseHashSet [#27690](https://github.com/ClickHouse/ClickHouse/pull/27690) ([Maksim Kita](https://github.com/kitaisreal)). +* Map data type parsing tests [#27692](https://github.com/ClickHouse/ClickHouse/pull/27692) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor arrayJoin check on partition expressions [#27733](https://github.com/ClickHouse/ClickHouse/pull/27733) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test 01014_lazy_database_concurrent_recreate_reattach_and_show_tables [#27734](https://github.com/ClickHouse/ClickHouse/pull/27734) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better code around decompression [2] [#27743](https://github.com/ClickHouse/ClickHouse/pull/27743) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Disable jemalloc under OSX [#27751](https://github.com/ClickHouse/ClickHouse/pull/27751) ([Raúl Marín](https://github.com/Algunenano)). +* try to collect some core dumps in perf tests [#27752](https://github.com/ClickHouse/ClickHouse/pull/27752) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix jemalloc under osx (zone_register() had been optimized out again) [#27753](https://github.com/ClickHouse/ClickHouse/pull/27753) ([Azat Khuzhin](https://github.com/azat)). +* Merging [#20089](https://github.com/ClickHouse/ClickHouse/issues/20089) [#27755](https://github.com/ClickHouse/ClickHouse/pull/27755) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix intersect/except with limit [#27757](https://github.com/ClickHouse/ClickHouse/pull/27757) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add HTTP string parsing test [#27762](https://github.com/ClickHouse/ClickHouse/pull/27762) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix some tests [#27785](https://github.com/ClickHouse/ClickHouse/pull/27785) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set function divide as suitable for short-circuit in case of Nullable(Decimal) [#27788](https://github.com/ClickHouse/ClickHouse/pull/27788) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unnecessary files [#27789](https://github.com/ClickHouse/ClickHouse/pull/27789) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Mark tests for `DatabaseReplicated` as green" [#27791](https://github.com/ClickHouse/ClickHouse/pull/27791) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove hardening for watches in DDLWorker [#27792](https://github.com/ClickHouse/ClickHouse/pull/27792) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Stateless test: Cleanup leftovers [#27793](https://github.com/ClickHouse/ClickHouse/pull/27793) ([Raúl Marín](https://github.com/Algunenano)). +* Dictionaries key types refactoring [#27795](https://github.com/ClickHouse/ClickHouse/pull/27795) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01822_short_circuit.reference (after merging [#27680](https://github.com/ClickHouse/ClickHouse/issues/27680)) [#27802](https://github.com/ClickHouse/ClickHouse/pull/27802) ([Azat Khuzhin](https://github.com/azat)). +* Proper shutdown global context [#27804](https://github.com/ClickHouse/ClickHouse/pull/27804) ([Amos Bird](https://github.com/amosbird)). +* 01766_todatetime64_no_timezone_arg: Use a date without timezone changes [#27810](https://github.com/ClickHouse/ClickHouse/pull/27810) ([Raúl Marín](https://github.com/Algunenano)). +* Use sessions more [#27817](https://github.com/ClickHouse/ClickHouse/pull/27817) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for clickhouse-keeper start after conversion [#27818](https://github.com/ClickHouse/ClickHouse/pull/27818) ([alesapin](https://github.com/alesapin)). +* Fix setting name "allow_experimental_database_materialized_postgresql" in the error message [#27824](https://github.com/ClickHouse/ClickHouse/pull/27824) ([Denny Crane](https://github.com/den-crane)). +* Fix bug in short-circuit found by fuzzer [#27826](https://github.com/ClickHouse/ClickHouse/pull/27826) ([Kruglov Pavel](https://github.com/Avogar)). +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix test 00443_preferred_block_size_bytes.sh [#27846](https://github.com/ClickHouse/ClickHouse/pull/27846) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Mute some integration tests until failures are fixed [#27862](https://github.com/ClickHouse/ClickHouse/pull/27862) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix bad cast in insertPostgreSQLValue [#27869](https://github.com/ClickHouse/ClickHouse/pull/27869) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add fuzzers for codecs [#27872](https://github.com/ClickHouse/ClickHouse/pull/27872) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove tmp folders from tests [#27878](https://github.com/ClickHouse/ClickHouse/pull/27878) ([Kruglov Pavel](https://github.com/Avogar)). +* blog article about perf tests [#27879](https://github.com/ClickHouse/ClickHouse/pull/27879) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* make the sql-standard window functions case insensitive [#27880](https://github.com/ClickHouse/ClickHouse/pull/27880) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Bump librdkafka (fixes use of an invalid/destroyed mutex) [#27883](https://github.com/ClickHouse/ClickHouse/pull/27883) ([Azat Khuzhin](https://github.com/azat)). +* fix decimal formatting settings in perf test [#27884](https://github.com/ClickHouse/ClickHouse/pull/27884) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add separate constants for interfaces LOCAL and TCP_INTERSERVER. [#27886](https://github.com/ClickHouse/ClickHouse/pull/27886) ([Vitaly Baranov](https://github.com/vitlibar)). +* Build fuzzers with clang-tidy [#27895](https://github.com/ClickHouse/ClickHouse/pull/27895) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow implicit cast bool to Field [#27921](https://github.com/ClickHouse/ClickHouse/pull/27921) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve server logs checking in integration tests [#27934](https://github.com/ClickHouse/ClickHouse/pull/27934) ([Azat Khuzhin](https://github.com/azat)). +* Get rid of mutable value in FunctionGetSetting. [#27982](https://github.com/ClickHouse/ClickHouse/pull/27982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable fuzzers build with clang-tidy [#27985](https://github.com/ClickHouse/ClickHouse/pull/27985) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Build fuzzers in CI [#27990](https://github.com/ClickHouse/ClickHouse/pull/27990) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix file progress for local [#27991](https://github.com/ClickHouse/ClickHouse/pull/27991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update libunwind [#27993](https://github.com/ClickHouse/ClickHouse/pull/27993) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix style typos [#28020](https://github.com/ClickHouse/ClickHouse/pull/28020) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Bump replxx [#28039](https://github.com/ClickHouse/ClickHouse/pull/28039) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398) [#28054](https://github.com/ClickHouse/ClickHouse/pull/28054) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix zookeeper secure client test [#28066](https://github.com/ClickHouse/ClickHouse/pull/28066) ([alesapin](https://github.com/alesapin)). +* Fix typo in docs [#28077](https://github.com/ClickHouse/ClickHouse/pull/28077) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed a typo in comments to `SinkToStorage` [#28078](https://github.com/ClickHouse/ClickHouse/pull/28078) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Use jinja template tests in fuzzer [#28079](https://github.com/ClickHouse/ClickHouse/pull/28079) ([Vladimir C](https://github.com/vdimir)). +* Clickhouse-keeper: renames and comments [#28080](https://github.com/ClickHouse/ClickHouse/pull/28080) ([alesapin](https://github.com/alesapin)). +* Update nanodbc [#28084](https://github.com/ClickHouse/ClickHouse/pull/28084) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve 01730_distributed_group_by_no_merge_order_by_long [#28123](https://github.com/ClickHouse/ClickHouse/pull/28123) ([Azat Khuzhin](https://github.com/azat)). +* Get rid of useless projection columns during merge [#28135](https://github.com/ClickHouse/ClickHouse/pull/28135) ([Amos Bird](https://github.com/amosbird)). +* Fix style [#28140](https://github.com/ClickHouse/ClickHouse/pull/28140) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Fix clickhouse keeper jepsen tests [#28143](https://github.com/ClickHouse/ClickHouse/pull/28143) ([alesapin](https://github.com/alesapin)). +* Updated ya.make files [#28157](https://github.com/ClickHouse/ClickHouse/pull/28157) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Better detection of the default interface in replicated fetches tests [#28184](https://github.com/ClickHouse/ClickHouse/pull/28184) ([alesapin](https://github.com/alesapin)). +* Reserve protocol number for ALTER PRIMARY KEY. [#28193](https://github.com/ClickHouse/ClickHouse/pull/28193) ([Amos Bird](https://github.com/amosbird)). +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Guard UDF container with a lock [#28211](https://github.com/ClickHouse/ClickHouse/pull/28211) ([Azat Khuzhin](https://github.com/azat)). +* Fix error codes [#28234](https://github.com/ClickHouse/ClickHouse/pull/28234) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries small fixes [#28249](https://github.com/ClickHouse/ClickHouse/pull/28249) ([Maksim Kita](https://github.com/kitaisreal)). +* Better nullable primary key implementation [#28269](https://github.com/ClickHouse/ClickHouse/pull/28269) ([Amos Bird](https://github.com/amosbird)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* test/stress: fix patterns for filtering out Raft messages [#28303](https://github.com/ClickHouse/ClickHouse/pull/28303) ([Azat Khuzhin](https://github.com/azat)). +* Rename system.views columns [#28319](https://github.com/ClickHouse/ClickHouse/pull/28319) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix arcadia build (generate ya.make) [#28326](https://github.com/ClickHouse/ClickHouse/pull/28326) ([DimasKovas](https://github.com/DimasKovas)). +* Try to fix arcadia build [#28333](https://github.com/ClickHouse/ClickHouse/pull/28333) ([DimasKovas](https://github.com/DimasKovas)). +* Fix test_storage_s3/test_put_get_with_globs (cleanup after test) [#28336](https://github.com/ClickHouse/ClickHouse/pull/28336) ([ianton-ru](https://github.com/ianton-ru)). +* Fix sed argument in test/fuzzer/run-fuzzer.sh [#28350](https://github.com/ClickHouse/ClickHouse/pull/28350) ([Vladimir C](https://github.com/vdimir)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Rework kafka topic creation. [#28354](https://github.com/ClickHouse/ClickHouse/pull/28354) ([Ilya Yatsishin](https://github.com/qoega)). +* Add compat between SinkToStorage and BlockOutputStream [#28361](https://github.com/ClickHouse/ClickHouse/pull/28361) ([DimasKovas](https://github.com/DimasKovas)). +* Try to fix arcadia build (generate ya.make) [#28382](https://github.com/ClickHouse/ClickHouse/pull/28382) ([DimasKovas](https://github.com/DimasKovas)). +* Add a test for a friend [#28396](https://github.com/ClickHouse/ClickHouse/pull/28396) ([alesapin](https://github.com/alesapin)). +* Update ya.make [#28403](https://github.com/ClickHouse/ClickHouse/pull/28403) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix disk with static files a little [#28411](https://github.com/ClickHouse/ClickHouse/pull/28411) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27980](https://github.com/ClickHouse/ClickHouse/issues/27980) [#28413](https://github.com/ClickHouse/ClickHouse/pull/28413) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01457_create_as_table_function_structure [#28428](https://github.com/ClickHouse/ClickHouse/pull/28428) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some rename tests [#28437](https://github.com/ClickHouse/ClickHouse/pull/28437) ([alesapin](https://github.com/alesapin)). +* Fix race in UDF (follow up) [#28438](https://github.com/ClickHouse/ClickHouse/pull/28438) ([Azat Khuzhin](https://github.com/azat)). +* Executable multiple pipes added test [#28503](https://github.com/ClickHouse/ClickHouse/pull/28503) ([Maksim Kita](https://github.com/kitaisreal)). +* UserDefinedFunctionFactory added comments [#28516](https://github.com/ClickHouse/ClickHouse/pull/28516) ([Maksim Kita](https://github.com/kitaisreal)). +* BorrowedObjectPool fix style [#28523](https://github.com/ClickHouse/ClickHouse/pull/28523) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for keeper 2 node configuration [#28526](https://github.com/ClickHouse/ClickHouse/pull/28526) ([alesapin](https://github.com/alesapin)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix race in zlib [#28534](https://github.com/ClickHouse/ClickHouse/pull/28534) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Bump poco to remove getpid() calls [#28537](https://github.com/ClickHouse/ClickHouse/pull/28537) ([Azat Khuzhin](https://github.com/azat)). +* Fix broken kafka test [#28542](https://github.com/ClickHouse/ClickHouse/pull/28542) ([alesapin](https://github.com/alesapin)). +* Fix format names in docs [#28557](https://github.com/ClickHouse/ClickHouse/pull/28557) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix wrong header of minmax_count projection [#28560](https://github.com/ClickHouse/ClickHouse/pull/28560) ([Amos Bird](https://github.com/amosbird)). +* remove recursion in ZstdInflatingReadBuffer [#28561](https://github.com/ClickHouse/ClickHouse/pull/28561) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Improve [C|T]SV errors [#28579](https://github.com/ClickHouse/ClickHouse/pull/28579) ([Raúl Marín](https://github.com/Algunenano)). +* Function dictGet small fix [#28615](https://github.com/ClickHouse/ClickHouse/pull/28615) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix arcadia build [#28640](https://github.com/ClickHouse/ClickHouse/pull/28640) ([DimasKovas](https://github.com/DimasKovas)). +* Add missed log level into TextLog [#28648](https://github.com/ClickHouse/ClickHouse/pull/28648) ([alesapin](https://github.com/alesapin)). +* Revert [#28082](https://github.com/ClickHouse/ClickHouse/issues/28082) [#28665](https://github.com/ClickHouse/ClickHouse/pull/28665) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert [#28397](https://github.com/ClickHouse/ClickHouse/issues/28397) [#28667](https://github.com/ClickHouse/ClickHouse/pull/28667) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.10.2.15-stable.md b/docs/changelogs/v21.10.2.15-stable.md index 05e278f03ae..d23ff95307c 100644 --- a/docs/changelogs/v21.10.2.15-stable.md +++ b/docs/changelogs/v21.10.2.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.2.15-stable FIXME as compared to v21.10.1.8013-prestable #### Improvement @@ -63,3 +70,17 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30182](https://github.com/ClickHouse/ClickHouse/pull/30182) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arcadia (pre)stable 21.10 build [#29250](https://github.com/ClickHouse/ClickHouse/pull/29250) ([DimasKovas](https://github.com/DimasKovas)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove unnecessary PEERDIR to libcxx-filesystem [#29798](https://github.com/ClickHouse/ClickHouse/pull/29798) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* SQL user defined functions fix alias [#30075](https://github.com/ClickHouse/ClickHouse/pull/30075) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.3.9-stable.md b/docs/changelogs/v21.10.3.9-stable.md index 78240367d55..8cccd01e77a 100644 --- a/docs/changelogs/v21.10.3.9-stable.md +++ b/docs/changelogs/v21.10.3.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.3.9-stable FIXME as compared to v21.10.2.15-stable #### New Feature @@ -43,3 +50,13 @@ * Backported in [#31255](https://github.com/ClickHouse/ClickHouse/issues/31255): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). * Backported in [#31436](https://github.com/ClickHouse/ClickHouse/issues/31436): Fix bug with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExecutable fix small issues [#30352](https://github.com/ClickHouse/ClickHouse/pull/30352) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). +* SQLUserDefinedFunctions composition fix [#30483](https://github.com/ClickHouse/ClickHouse/pull/30483) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix typo in USE_MYSQL check [#31226](https://github.com/ClickHouse/ClickHouse/pull/31226) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.4.26-stable.md b/docs/changelogs/v21.10.4.26-stable.md index b2089fad0c4..06eab149e6c 100644 --- a/docs/changelogs/v21.10.4.26-stable.md +++ b/docs/changelogs/v21.10.4.26-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.4.26-stable FIXME as compared to v21.10.3.9-stable #### Performance Improvement @@ -23,3 +30,7 @@ * Backported in [#31939](https://github.com/ClickHouse/ClickHouse/issues/31939): - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). * Backported in [#31909](https://github.com/ClickHouse/ClickHouse/issues/31909): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.10.5.3-stable.md b/docs/changelogs/v21.10.5.3-stable.md index a591c4a07a8..7a87b349324 100644 --- a/docs/changelogs/v21.10.5.3-stable.md +++ b/docs/changelogs/v21.10.5.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.5.3-stable FIXME as compared to v21.10.4.26-stable #### Bug Fix diff --git a/docs/changelogs/v21.10.6.2-stable.md b/docs/changelogs/v21.10.6.2-stable.md index da146ee364d..c9afcfae75e 100644 --- a/docs/changelogs/v21.10.6.2-stable.md +++ b/docs/changelogs/v21.10.6.2-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.6.2-stable FIXME as compared to v21.10.5.3-stable #### Bug Fix @@ -20,3 +27,13 @@ * Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md index 2aab0293223..95c982a2a2e 100644 --- a/docs/changelogs/v21.11.1.8636-prestable.md +++ b/docs/changelogs/v21.11.1.8636-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.1.8636-prestable FIXME as compared to v21.10.1.8013-prestable #### Backward Incompatible Change @@ -13,7 +20,7 @@ * Users now can set comments to database in `CREATE DATABASE` statement ... [#29429](https://github.com/ClickHouse/ClickHouse/pull/29429) ([Vasily Nemkov](https://github.com/Enmk)). * New function` mapContainsKeyLike` to get the map that key matches a simple regular expression. [#29471](https://github.com/ClickHouse/ClickHouse/pull/29471) ([凌涛](https://github.com/lingtaolf)). * Huawei OBS Storage support. Closes [#24294](https://github.com/ClickHouse/ClickHouse/issues/24294). [#29511](https://github.com/ClickHouse/ClickHouse/pull/29511) ([kevin wan](https://github.com/MaxWk)). -* ClickHouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). +* Clickhouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). * - Added MD4 and SHA384 functions. [#29602](https://github.com/ClickHouse/ClickHouse/pull/29602) ([Nikita Tikhomirov](https://github.com/NSTikhomirov)). * Support EXISTS(subquery). Closes [#6852](https://github.com/ClickHouse/ClickHouse/issues/6852). [#29731](https://github.com/ClickHouse/ClickHouse/pull/29731) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added function `ngram`. Closes [#29699](https://github.com/ClickHouse/ClickHouse/issues/29699). [#29738](https://github.com/ClickHouse/ClickHouse/pull/29738) ([Maksim Kita](https://github.com/kitaisreal)). @@ -190,6 +197,279 @@ * NO CL ENTRY: 'Revert "Revert "Improve usability of `remote_url_allow_hosts`""'. [#30708](https://github.com/ClickHouse/ClickHouse/pull/30708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * NO CL ENTRY: 'remove some unneeded header files'. [#30722](https://github.com/ClickHouse/ClickHouse/pull/30722) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix race in StorageSystemRocksDB [#29289](https://github.com/ClickHouse/ClickHouse/pull/29289) ([Vladimir C](https://github.com/vdimir)). +* Fixed logging level for message in `S3Common.cpp` [#29308](https://github.com/ClickHouse/ClickHouse/pull/29308) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Revert parse number binary literal [#29309](https://github.com/ClickHouse/ClickHouse/pull/29309) ([Maksim Kita](https://github.com/kitaisreal)). +* Parser number binary literal update [#29310](https://github.com/ClickHouse/ClickHouse/pull/29310) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix assertion in DDLDependencyVisitor [#29323](https://github.com/ClickHouse/ClickHouse/pull/29323) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Function h3GetFaces optimization [#29335](https://github.com/ClickHouse/ClickHouse/pull/29335) ([Maksim Kita](https://github.com/kitaisreal)). +* Less sleeps in integration tests. [#29338](https://github.com/ClickHouse/ClickHouse/pull/29338) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix trivial mistake in DiskWebServer [#29340](https://github.com/ClickHouse/ClickHouse/pull/29340) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration test for RocksDB [#29341](https://github.com/ClickHouse/ClickHouse/pull/29341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in comment [#29342](https://github.com/ClickHouse/ClickHouse/pull/29342) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add lld into Build-Depends for debian package [#29343](https://github.com/ClickHouse/ClickHouse/pull/29343) ([Azat Khuzhin](https://github.com/azat)). +* Apply a patch from Azat [#29344](https://github.com/ClickHouse/ClickHouse/pull/29344) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update libhdfs3 [#29345](https://github.com/ClickHouse/ClickHouse/pull/29345) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove ya.make [#29346](https://github.com/ClickHouse/ClickHouse/pull/29346) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert in table function `merge` with database regexp [#29355](https://github.com/ClickHouse/ClickHouse/pull/29355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless files related to pytest [#29361](https://github.com/ClickHouse/ClickHouse/pull/29361) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete non-automated tests [#29362](https://github.com/ClickHouse/ClickHouse/pull/29362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration test [#29365](https://github.com/ClickHouse/ClickHouse/pull/29365) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix H3 function name (h3ResIsClassIII to h3IsResClassIII) [#29368](https://github.com/ClickHouse/ClickHouse/pull/29368) ([Bharat Nallan](https://github.com/bharatnc)). +* RFC: tests: purge unused configs [#29375](https://github.com/ClickHouse/ClickHouse/pull/29375) ([Azat Khuzhin](https://github.com/azat)). +* Fix ugly typo [#29379](https://github.com/ClickHouse/ClickHouse/pull/29379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix comments in AggregateFunctionFactory [#29380](https://github.com/ClickHouse/ClickHouse/pull/29380) ([Bharat Nallan](https://github.com/bharatnc)). +* Add GraphCDN in adopter list [#29387](https://github.com/ClickHouse/ClickHouse/pull/29387) ([Mohamad Fadhil](https://github.com/sdil)). +* Fix 02015_async_inserts_2 flakiness [#29390](https://github.com/ClickHouse/ClickHouse/pull/29390) ([Azat Khuzhin](https://github.com/azat)). +* test for [#23634](https://github.com/ClickHouse/ClickHouse/issues/23634) ( nullable PK and negate cond ) [#29392](https://github.com/ClickHouse/ClickHouse/pull/29392) ([Denny Crane](https://github.com/den-crane)). +* Merge EmbeddedRocksDBBlockInputStream and EmbeddedRocksDBSource [#29428](https://github.com/ClickHouse/ClickHouse/pull/29428) ([Vladimir C](https://github.com/vdimir)). +* Map bloom filter index mapValues equals function support [#29431](https://github.com/ClickHouse/ClickHouse/pull/29431) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary Array nested Map added test [#29432](https://github.com/ClickHouse/ClickHouse/pull/29432) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary view different database added test [#29443](https://github.com/ClickHouse/ClickHouse/pull/29443) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix deadlock in concurrent async inserts and truncates [#29444](https://github.com/ClickHouse/ClickHouse/pull/29444) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky test 01158_zookeeper_log_long [#29445](https://github.com/ClickHouse/ClickHouse/pull/29445) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: fix long tag check for flaky check (--test-runs > 1) [#29449](https://github.com/ClickHouse/ClickHouse/pull/29449) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: fix shared list object (by fixing manager lifetime) [#29452](https://github.com/ClickHouse/ClickHouse/pull/29452) ([Azat Khuzhin](https://github.com/azat)). +* Enable memory profiler on CI [#29453](https://github.com/ClickHouse/ClickHouse/pull/29453) ([Azat Khuzhin](https://github.com/azat)). +* test for [#23634](https://github.com/ClickHouse/ClickHouse/issues/23634) added tests for tuples [#29460](https://github.com/ClickHouse/ClickHouse/pull/29460) ([Denny Crane](https://github.com/den-crane)). +* Fix 2024_merge_regexp_assert [#29461](https://github.com/ClickHouse/ClickHouse/pull/29461) ([Azat Khuzhin](https://github.com/azat)). +* Add std::cerr/std::cout style check [#29464](https://github.com/ClickHouse/ClickHouse/pull/29464) ([Azat Khuzhin](https://github.com/azat)). +* [github] we're switching to CLA based on Apache CLA [#29466](https://github.com/ClickHouse/ClickHouse/pull/29466) ([ClickHouse Admin](https://github.com/clickhouse-admin)). +* Bloom filter indexes updated tests [#29474](https://github.com/ClickHouse/ClickHouse/pull/29474) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove more data streams [#29491](https://github.com/ClickHouse/ClickHouse/pull/29491) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible hang in PushingAsyncPipelineExecutor. [#29494](https://github.com/ClickHouse/ClickHouse/pull/29494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use `max_parse_depth` in fuzzers [#29497](https://github.com/ClickHouse/ClickHouse/pull/29497) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Introducing Fn concept for function signature checking, simplifying SimpleCache [#29500](https://github.com/ClickHouse/ClickHouse/pull/29500) ([Mike Kot](https://github.com/myrrc)). +* Fix LOGICAL_ERROR for INSERT with concurrent ALTER [#29502](https://github.com/ClickHouse/ClickHouse/pull/29502) ([Azat Khuzhin](https://github.com/azat)). +* Log queries to external databases (since they may be rewritten) [#29503](https://github.com/ClickHouse/ClickHouse/pull/29503) ([Azat Khuzhin](https://github.com/azat)). +* Fix server pid (hence exit code and attaching with gdb) in fuzzer tests [#29513](https://github.com/ClickHouse/ClickHouse/pull/29513) ([Azat Khuzhin](https://github.com/azat)). +* Bump cmake minimum required version to 3.14 [#29515](https://github.com/ClickHouse/ClickHouse/pull/29515) ([Azat Khuzhin](https://github.com/azat)). +* Changelog heredoc added backward incompatible change [#29530](https://github.com/ClickHouse/ClickHouse/pull/29530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix test names [#29539](https://github.com/ClickHouse/ClickHouse/pull/29539) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Making Monotonicity an aggregate to use with designated initializers [#29540](https://github.com/ClickHouse/ClickHouse/pull/29540) ([Mike Kot](https://github.com/myrrc)). +* Generalize code in IColumn::permute [#29545](https://github.com/ClickHouse/ClickHouse/pull/29545) ([Anton Popov](https://github.com/CurtizJ)). +* Add error for multiple keys without current_key_id [#29546](https://github.com/ClickHouse/ClickHouse/pull/29546) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* support conversion between date32 and datetime64 [#29552](https://github.com/ClickHouse/ClickHouse/pull/29552) ([kevin wan](https://github.com/MaxWk)). +* Fixed test with dictGet without database name as column default value [#29568](https://github.com/ClickHouse/ClickHouse/pull/29568) ([Maksim Kita](https://github.com/kitaisreal)). +* Add FAIL message to test_results.tsv [#29583](https://github.com/ClickHouse/ClickHouse/pull/29583) ([Dmitry Novik](https://github.com/novikd)). +* Generalize code in `IColumn::updatePermutation` [#29595](https://github.com/ClickHouse/ClickHouse/pull/29595) ([Anton Popov](https://github.com/CurtizJ)). +* Add fuzzer for `executeQuery` function [#29596](https://github.com/ClickHouse/ClickHouse/pull/29596) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix docker/test/util declaration [#29597](https://github.com/ClickHouse/ClickHouse/pull/29597) ([Dmitry Novik](https://github.com/novikd)). +* Skip test for executable table function under MSan [#29600](https://github.com/ClickHouse/ClickHouse/pull/29600) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve some more integration tests [#29603](https://github.com/ClickHouse/ClickHouse/pull/29603) ([Ilya Yatsishin](https://github.com/qoega)). +* Improve CompressedReadBuffer [#29605](https://github.com/ClickHouse/ClickHouse/pull/29605) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not send many signals at server restart (integration tests). [#29608](https://github.com/ClickHouse/ClickHouse/pull/29608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between MergeTask and storage destruction [#29614](https://github.com/ClickHouse/ClickHouse/pull/29614) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* BloomFilter indexes map data type support fix unsafe identifier cast [#29623](https://github.com/ClickHouse/ClickHouse/pull/29623) ([Maksim Kita](https://github.com/kitaisreal)). +* Follow-up for [#26231](https://github.com/ClickHouse/ClickHouse/issues/26231) [#29626](https://github.com/ClickHouse/ClickHouse/pull/29626) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Map get non const key added test [#29636](https://github.com/ClickHouse/ClickHouse/pull/29636) ([Maksim Kita](https://github.com/kitaisreal)). +* Add columns in columns.sql for INFORMATION_SCHEMA [#29637](https://github.com/ClickHouse/ClickHouse/pull/29637) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Merging [#29376](https://github.com/ClickHouse/ClickHouse/issues/29376) [#29649](https://github.com/ClickHouse/ClickHouse/pull/29649) ([Anton Popov](https://github.com/CurtizJ)). +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not try to fuzz `USE`/`SET` queries and print stacktrace [#29660](https://github.com/ClickHouse/ClickHouse/pull/29660) ([Azat Khuzhin](https://github.com/azat)). +* Rename `common` to `base` [#29661](https://github.com/ClickHouse/ClickHouse/pull/29661) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move modelEvaluate function to its own file [#29662](https://github.com/ClickHouse/ClickHouse/pull/29662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Perf fixes [#29663](https://github.com/ClickHouse/ClickHouse/pull/29663) ([Azat Khuzhin](https://github.com/azat)). +* UserDefinedExecutableFunctions added implicit cast [#29666](https://github.com/ClickHouse/ClickHouse/pull/29666) ([Maksim Kita](https://github.com/kitaisreal)). +* Cleanup common defines [#29680](https://github.com/ClickHouse/ClickHouse/pull/29680) ([Azat Khuzhin](https://github.com/azat)). +* Fix Xcode 13 build [#29682](https://github.com/ClickHouse/ClickHouse/pull/29682) ([Denis Glazachev](https://github.com/traceon)). +* Non-recursive implementation for type list and its functions [#29683](https://github.com/ClickHouse/ClickHouse/pull/29683) ([Mike Kot](https://github.com/myrrc)). +* Reorganize contrib/ IDE folders [#29684](https://github.com/ClickHouse/ClickHouse/pull/29684) ([Denis Glazachev](https://github.com/traceon)). +* Cleanup unbundled image [#29689](https://github.com/ClickHouse/ClickHouse/pull/29689) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory tracking for merges and mutations [#29691](https://github.com/ClickHouse/ClickHouse/pull/29691) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race in WriteIndirectBuffer (used in DiskMemory) [#29692](https://github.com/ClickHouse/ClickHouse/pull/29692) ([Azat Khuzhin](https://github.com/azat)). +* Fix flacky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)). +* BorrowedObjectPool condition variable notify fix [#29722](https://github.com/ClickHouse/ClickHouse/pull/29722) ([Maksim Kita](https://github.com/kitaisreal)). +* Better exception message for local interactive [#29737](https://github.com/ClickHouse/ClickHouse/pull/29737) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix --stage for clickhouse-local [#29745](https://github.com/ClickHouse/ClickHouse/pull/29745) ([Azat Khuzhin](https://github.com/azat)). +* Forbid Nullable for JSONExtract* (JSONExtract() still supports Nullable), leads to SIGSEGV before [#29746](https://github.com/ClickHouse/ClickHouse/pull/29746) ([Azat Khuzhin](https://github.com/azat)). +* TableFunctionDictionary fix comment [#29747](https://github.com/ClickHouse/ClickHouse/pull/29747) ([Maksim Kita](https://github.com/kitaisreal)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some merging streams [#29768](https://github.com/ClickHouse/ClickHouse/pull/29768) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Parse query from metadata exception throw fix [#29779](https://github.com/ClickHouse/ClickHouse/pull/29779) ([Maksim Kita](https://github.com/kitaisreal)). +* Simplify obtaining of server pid in fuzzer tests [#29781](https://github.com/ClickHouse/ClickHouse/pull/29781) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race between fatal error handler and progress packets [#29783](https://github.com/ClickHouse/ClickHouse/pull/29783) ([Azat Khuzhin](https://github.com/azat)). +* Fix parallel_view_processing [#29786](https://github.com/ClickHouse/ClickHouse/pull/29786) ([Azat Khuzhin](https://github.com/azat)). +* Function reinterpretAs improve readability [#29796](https://github.com/ClickHouse/ClickHouse/pull/29796) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor ConcurrentBoundedQueue [#29801](https://github.com/ClickHouse/ClickHouse/pull/29801) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve usability of error messages when error is caused by sophisticated interventions [#29803](https://github.com/ClickHouse/ClickHouse/pull/29803) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvement for [#28373](https://github.com/ClickHouse/ClickHouse/issues/28373) [#29804](https://github.com/ClickHouse/ClickHouse/pull/29804) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better diagnostic for OPTIMIZE [#29812](https://github.com/ClickHouse/ClickHouse/pull/29812) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add listen_backlog to documentation [#29813](https://github.com/ClickHouse/ClickHouse/pull/29813) ([Azat Khuzhin](https://github.com/azat)). +* Dictionary attributes updated documentation [#29816](https://github.com/ClickHouse/ClickHouse/pull/29816) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix compilation with glibc 2.34 (MINSIGSTKSZ defined as sysconf(_SC_SIGSTKSZ)) [#29820](https://github.com/ClickHouse/ClickHouse/pull/29820) ([Azat Khuzhin](https://github.com/azat)). +* Make memory_profiler_step API cleaner [#29825](https://github.com/ClickHouse/ClickHouse/pull/29825) ([Azat Khuzhin](https://github.com/azat)). +* Add coroutines example. [#29827](https://github.com/ClickHouse/ClickHouse/pull/29827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add coroutines example. [#29841](https://github.com/ClickHouse/ClickHouse/pull/29841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update changelog to include caveats about Nullable types in data skipping indexes [#29855](https://github.com/ClickHouse/ClickHouse/pull/29855) ([Azat Khuzhin](https://github.com/azat)). +* Rewrite clickhouse-test to use python clickhouse_driver [#29856](https://github.com/ClickHouse/ClickHouse/pull/29856) ([Azat Khuzhin](https://github.com/azat)). +* Fix client [#29864](https://github.com/ClickHouse/ClickHouse/pull/29864) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some more streams. [#29898](https://github.com/ClickHouse/ClickHouse/pull/29898) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add logging in ZooKeeper client [#29901](https://github.com/ClickHouse/ClickHouse/pull/29901) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix some flacky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Grep server log even if it contains binary data [#29903](https://github.com/ClickHouse/ClickHouse/pull/29903) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Cosmetic refactoring of server constants. [#29913](https://github.com/ClickHouse/ClickHouse/pull/29913) ([Amos Bird](https://github.com/amosbird)). +* Format improvement of AlterQuery [#29916](https://github.com/ClickHouse/ClickHouse/pull/29916) ([flynn](https://github.com/ucasfl)). +* Fix flaky integration tests (test_backup_restore/test_input_format_parallel_parsing_memory_tracking) [#29919](https://github.com/ClickHouse/ClickHouse/pull/29919) ([Azat Khuzhin](https://github.com/azat)). +* Fix some flaky tests [#29923](https://github.com/ClickHouse/ClickHouse/pull/29923) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix build due to conflicts in serverConstants [#29931](https://github.com/ClickHouse/ClickHouse/pull/29931) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes [#29932](https://github.com/ClickHouse/ClickHouse/pull/29932) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash from SentryWriter [#29933](https://github.com/ClickHouse/ClickHouse/pull/29933) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `printf` function usage. [#29935](https://github.com/ClickHouse/ClickHouse/pull/29935) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Stateless flaky tests fixes [#29947](https://github.com/ClickHouse/ClickHouse/pull/29947) ([Azat Khuzhin](https://github.com/azat)). +* Fix filtering by tuple (some conditions was lost during analyzing) [#29956](https://github.com/ClickHouse/ClickHouse/pull/29956) ([Azat Khuzhin](https://github.com/azat)). +* Allow memory profiler under sanitizers (ASan/UBsan/MSan only) [#29979](https://github.com/ClickHouse/ClickHouse/pull/29979) ([Azat Khuzhin](https://github.com/azat)). +* Tests naming fix [#29982](https://github.com/ClickHouse/ClickHouse/pull/29982) ([Maksim Kita](https://github.com/kitaisreal)). +* More timeouts in test scripts [#29992](https://github.com/ClickHouse/ClickHouse/pull/29992) ([alesapin](https://github.com/alesapin)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Remove streams from formats. [#30001](https://github.com/ClickHouse/ClickHouse/pull/30001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow-up to [#29901](https://github.com/ClickHouse/ClickHouse/issues/29901) [#30003](https://github.com/ClickHouse/ClickHouse/pull/30003) ([Alexander Tokmakov](https://github.com/tavplubix)). +* More strict check for intersecting parts [#30005](https://github.com/ClickHouse/ClickHouse/pull/30005) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Ignore parallel removing warning in 00992_system_parts_race_condition_zookeeper_long [#30007](https://github.com/ClickHouse/ClickHouse/pull/30007) ([Azat Khuzhin](https://github.com/azat)). +* Try remove excessive logging [#30008](https://github.com/ClickHouse/ClickHouse/pull/30008) ([Alexander Tokmakov](https://github.com/tavplubix)). +* remove redundant dot in exception message [#30017](https://github.com/ClickHouse/ClickHouse/pull/30017) ([flynn](https://github.com/ucasfl)). +* Fix build [#30028](https://github.com/ClickHouse/ClickHouse/pull/30028) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Attach trace_log to stateless tests report as is [#30030](https://github.com/ClickHouse/ClickHouse/pull/30030) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove unused headers and handle exception 'unrecognised option' in clickhouse-local [#30053](https://github.com/ClickHouse/ClickHouse/pull/30053) ([Kruglov Pavel](https://github.com/Avogar)). +* clickhouse-test: replace clickhouse-driver with http interface (via http.client) [#30065](https://github.com/ClickHouse/ClickHouse/pull/30065) ([Azat Khuzhin](https://github.com/azat)). +* Fix hardware utilization info printing in client [#30072](https://github.com/ClickHouse/ClickHouse/pull/30072) ([Dmitry Novik](https://github.com/novikd)). +* cmake generator: unlink before creating a link [#30073](https://github.com/ClickHouse/ClickHouse/pull/30073) ([lehasm](https://github.com/lehasm)). +* SQL user defined functions fix alias [#30075](https://github.com/ClickHouse/ClickHouse/pull/30075) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Greenhouse careers page to website [#30077](https://github.com/ClickHouse/ClickHouse/pull/30077) ([Cody Baker](https://github.com/codyrobert)). +* Update team photos on website [#30078](https://github.com/ClickHouse/ClickHouse/pull/30078) ([Cody Baker](https://github.com/codyrobert)). +* Add webinar signup promo to website homepage [#30079](https://github.com/ClickHouse/ClickHouse/pull/30079) ([Cody Baker](https://github.com/codyrobert)). +* test for rename atomic hanging [#30080](https://github.com/ClickHouse/ClickHouse/pull/30080) ([Denny Crane](https://github.com/den-crane)). +* Modify ConnectionPoolWithFailover get_priority ROUND_ROBIN comments [#30092](https://github.com/ClickHouse/ClickHouse/pull/30092) ([小路](https://github.com/nicelulu)). +* Fix flaky test 01939_network_send_bytes_metrics [#30134](https://github.com/ClickHouse/ClickHouse/pull/30134) ([Dmitry Novik](https://github.com/novikd)). +* System data skipping indices size fix test names [#30141](https://github.com/ClickHouse/ClickHouse/pull/30141) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionsJSON avoid copying object element during iteration [#30145](https://github.com/ClickHouse/ClickHouse/pull/30145) ([Maksim Kita](https://github.com/kitaisreal)). +* Disable fsync_metadata on CI [#30149](https://github.com/ClickHouse/ClickHouse/pull/30149) ([Azat Khuzhin](https://github.com/azat)). +* Make test_MemoryTracking::test_http not flaky [#30150](https://github.com/ClickHouse/ClickHouse/pull/30150) ([Azat Khuzhin](https://github.com/azat)). +* Add careers menu item to navigation [#30151](https://github.com/ClickHouse/ClickHouse/pull/30151) ([Cody Baker](https://github.com/codyrobert)). +* Update yandex logo on homepage [#30152](https://github.com/ClickHouse/ClickHouse/pull/30152) ([Cody Baker](https://github.com/codyrobert)). +* Remove stream interfaces [#30171](https://github.com/ClickHouse/ClickHouse/pull/30171) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better interfaces for `IDataType` and `ISerialization` [#30174](https://github.com/ClickHouse/ClickHouse/pull/30174) ([Anton Popov](https://github.com/CurtizJ)). +* Add blog post for v21.10 release [#30186](https://github.com/ClickHouse/ClickHouse/pull/30186) ([Cody Baker](https://github.com/codyrobert)). +* Smaller smoothing window in throttler. [#30193](https://github.com/ClickHouse/ClickHouse/pull/30193) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix clang-tidy warnings in FunctionsJSON code [#30217](https://github.com/ClickHouse/ClickHouse/pull/30217) ([Azat Khuzhin](https://github.com/azat)). +* Fix --hung-check in clickhouse-test [#30218](https://github.com/ClickHouse/ClickHouse/pull/30218) ([Azat Khuzhin](https://github.com/azat)). +* FunctionsJSON updated [#30228](https://github.com/ClickHouse/ClickHouse/pull/30228) ([Maksim Kita](https://github.com/kitaisreal)). +* PolygonDictionary fix bytes_allocated [#30239](https://github.com/ClickHouse/ClickHouse/pull/30239) ([Maksim Kita](https://github.com/kitaisreal)). +* ComplexKeyHashedDictionary fix keys copy [#30241](https://github.com/ClickHouse/ClickHouse/pull/30241) ([Maksim Kita](https://github.com/kitaisreal)). +* Removing data streams folder [#30247](https://github.com/ClickHouse/ClickHouse/pull/30247) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* --stage for clickhouse-local [#30275](https://github.com/ClickHouse/ClickHouse/pull/30275) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use forward declaration for Buffer<> in generic headers [#30276](https://github.com/ClickHouse/ClickHouse/pull/30276) ([Azat Khuzhin](https://github.com/azat)). +* Less threads in clickhouse-local, fix Ok. printing [#30282](https://github.com/ClickHouse/ClickHouse/pull/30282) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Adjust resource heavy tests [#30284](https://github.com/ClickHouse/ClickHouse/pull/30284) ([Raúl Marín](https://github.com/Algunenano)). +* Fix printing stacktraces for clickhouse-local [#30285](https://github.com/ClickHouse/ClickHouse/pull/30285) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throwing syntax exception in clickhouse-local [#30288](https://github.com/ClickHouse/ClickHouse/pull/30288) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Various fixes to install procedure [#30295](https://github.com/ClickHouse/ClickHouse/pull/30295) ([Denis Glazachev](https://github.com/traceon)). +* Fix clickhouse-local break on timeout [#30297](https://github.com/ClickHouse/ClickHouse/pull/30297) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add RISC-V build [#30298](https://github.com/ClickHouse/ClickHouse/pull/30298) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move SquashingTransform to Interpreters (to fix split build) [#30318](https://github.com/ClickHouse/ClickHouse/pull/30318) ([Azat Khuzhin](https://github.com/azat)). +* Increase default wait of the server start in clickhouse-test [#30320](https://github.com/ClickHouse/ClickHouse/pull/30320) ([Azat Khuzhin](https://github.com/azat)). +* Update memory optimisation for MergingSorted. [#30322](https://github.com/ClickHouse/ClickHouse/pull/30322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Single sorting step [#30335](https://github.com/ClickHouse/ClickHouse/pull/30335) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Send table columns in clickhouse-local [#30336](https://github.com/ClickHouse/ClickHouse/pull/30336) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix consumed memory indication in clickhouse-client [#30337](https://github.com/ClickHouse/ClickHouse/pull/30337) ([Dmitry Novik](https://github.com/novikd)). +* Fix fuzzer build [#30344](https://github.com/ClickHouse/ClickHouse/pull/30344) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* ExecutablePool dictionary source fix borrow timeout milliseconds [#30345](https://github.com/ClickHouse/ClickHouse/pull/30345) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExecutable fix small issues [#30352](https://github.com/ClickHouse/ClickHouse/pull/30352) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). +* Update obsolete comments. [#30359](https://github.com/ClickHouse/ClickHouse/pull/30359) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Rename readWkt* functions to readWKT*, added an alias SVG for svg function [#30361](https://github.com/ClickHouse/ClickHouse/pull/30361) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash when minmax_count projection is used with primary key in partition expr [#30384](https://github.com/ClickHouse/ClickHouse/pull/30384) ([Amos Bird](https://github.com/amosbird)). +* Follow-up for [#30282](https://github.com/ClickHouse/ClickHouse/issues/30282) [#30412](https://github.com/ClickHouse/ClickHouse/pull/30412) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ExecutableUDF example [#30436](https://github.com/ClickHouse/ClickHouse/pull/30436) ([Maksim Kita](https://github.com/kitaisreal)). +* Use robot token in actions for statuses [#30439](https://github.com/ClickHouse/ClickHouse/pull/30439) ([alesapin](https://github.com/alesapin)). +* Remove statuses from actions [#30444](https://github.com/ClickHouse/ClickHouse/pull/30444) ([alesapin](https://github.com/alesapin)). +* Fix s3 for github actions [#30447](https://github.com/ClickHouse/ClickHouse/pull/30447) ([alesapin](https://github.com/alesapin)). +* ExecutableUDF example fix style check [#30451](https://github.com/ClickHouse/ClickHouse/pull/30451) ([Maksim Kita](https://github.com/kitaisreal)). +* Support VALUES format in async inserts [#30456](https://github.com/ClickHouse/ClickHouse/pull/30456) ([Anton Popov](https://github.com/CurtizJ)). +* Update release date and add training link [#30475](https://github.com/ClickHouse/ClickHouse/pull/30475) ([Cody Baker](https://github.com/codyrobert)). +* Fix horizontal scroll bar [#30476](https://github.com/ClickHouse/ClickHouse/pull/30476) ([Cody Baker](https://github.com/codyrobert)). +* SQLUserDefinedFunctions composition fix [#30483](https://github.com/ClickHouse/ClickHouse/pull/30483) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying builds on github actions [#30493](https://github.com/ClickHouse/ClickHouse/pull/30493) ([alesapin](https://github.com/alesapin)). +* HashedArrayDictionary optimize read multiple attributes [#30501](https://github.com/ClickHouse/ClickHouse/pull/30501) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix style regressions on benchmark page [#30506](https://github.com/ClickHouse/ClickHouse/pull/30506) ([Cody Baker](https://github.com/codyrobert)). +* Change link in nav from community -> learn [#30508](https://github.com/ClickHouse/ClickHouse/pull/30508) ([Cody Baker](https://github.com/codyrobert)). +* Add placeholder to play.html inputs [#30509](https://github.com/ClickHouse/ClickHouse/pull/30509) ([Vitaly Orlov](https://github.com/orloffv)). +* Add bytes to stats and human readable rows to play.html [#30511](https://github.com/ClickHouse/ClickHouse/pull/30511) ([Vitaly Orlov](https://github.com/orloffv)). +* clickhouse-local: fix block lost in interactive mode [#30521](https://github.com/ClickHouse/ClickHouse/pull/30521) ([Azat Khuzhin](https://github.com/azat)). +* Remove check_columns argument from MergeTree code (false was never passed) [#30522](https://github.com/ClickHouse/ClickHouse/pull/30522) ([Azat Khuzhin](https://github.com/azat)). +* Fix typo and update NuRaft [#30550](https://github.com/ClickHouse/ClickHouse/pull/30550) ([alesapin](https://github.com/alesapin)). +* Refactoring in codec encrypted [#30564](https://github.com/ClickHouse/ClickHouse/pull/30564) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Try to fix [#30397](https://github.com/ClickHouse/ClickHouse/issues/30397) [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support Nullable(String) type in arrayStringConcat [#30566](https://github.com/ClickHouse/ClickHouse/pull/30566) ([Nikita Taranov](https://github.com/nickitat)). +* Function ngrams fixed tests [#30567](https://github.com/ClickHouse/ClickHouse/pull/30567) ([Maksim Kita](https://github.com/kitaisreal)). +* Test clickhouse local columns description fix number [#30568](https://github.com/ClickHouse/ClickHouse/pull/30568) ([Maksim Kita](https://github.com/kitaisreal)). +* Update documentation for distributed_push_down_limit [#30577](https://github.com/ClickHouse/ClickHouse/pull/30577) ([Azat Khuzhin](https://github.com/azat)). +* Fix tests that relies on checking stack size under TSan [#30579](https://github.com/ClickHouse/ClickHouse/pull/30579) ([Azat Khuzhin](https://github.com/azat)). +* Add metadata for *_log into tests artifacts [#30589](https://github.com/ClickHouse/ClickHouse/pull/30589) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR on connection draining in case of ECONNRESET [#30594](https://github.com/ClickHouse/ClickHouse/pull/30594) ([Azat Khuzhin](https://github.com/azat)). +* Adjust perf test for simdjson [#30596](https://github.com/ClickHouse/ClickHouse/pull/30596) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update minimum allowed compiler versions [#30597](https://github.com/ClickHouse/ClickHouse/pull/30597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace estimated_size with optional for readFile/createReadBufferFromFileBase [#30611](https://github.com/ClickHouse/ClickHouse/pull/30611) ([Azat Khuzhin](https://github.com/azat)). +* RFC: Relax minimal clang version (set it to 12, was 13) [#30613](https://github.com/ClickHouse/ClickHouse/pull/30613) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: increase delay for initial SELECT 1 check [#30619](https://github.com/ClickHouse/ClickHouse/pull/30619) ([Azat Khuzhin](https://github.com/azat)). +* Update Client.cpp [#30636](https://github.com/ClickHouse/ClickHouse/pull/30636) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update LocalServer.cpp [#30637](https://github.com/ClickHouse/ClickHouse/pull/30637) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Typo fix in play.html [#30638](https://github.com/ClickHouse/ClickHouse/pull/30638) ([Vitaly Orlov](https://github.com/orloffv)). +* Fix argument types for now and now64 [#30639](https://github.com/ClickHouse/ClickHouse/pull/30639) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix intersecting parts in `parts_to_do` [#30651](https://github.com/ClickHouse/ClickHouse/pull/30651) ([Alexander Tokmakov](https://github.com/tavplubix)). +* HashedDictionaries added read performance tests [#30653](https://github.com/ClickHouse/ClickHouse/pull/30653) ([Maksim Kita](https://github.com/kitaisreal)). +* Change `alter_lock` from `RWLock` to `std::timed_mutex` [#30658](https://github.com/ClickHouse/ClickHouse/pull/30658) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Initialize custom TLDs in clickhouse-local [#30660](https://github.com/ClickHouse/ClickHouse/pull/30660) ([Azat Khuzhin](https://github.com/azat)). +* Add QueryProfilerRuns profile event [#30661](https://github.com/ClickHouse/ClickHouse/pull/30661) ([Azat Khuzhin](https://github.com/azat)). +* Switch everything left from `` to `` [#30662](https://github.com/ClickHouse/ClickHouse/pull/30662) ([Azat Khuzhin](https://github.com/azat)). +* Improve usability of `remote_url_allow_hosts` [#30673](https://github.com/ClickHouse/ClickHouse/pull/30673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix indecency [#30337](https://github.com/ClickHouse/ClickHouse/issues/30337) [#30674](https://github.com/ClickHouse/ClickHouse/pull/30674) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better wait for server start in integration tests [#30677](https://github.com/ClickHouse/ClickHouse/pull/30677) ([alesapin](https://github.com/alesapin)). +* Fix keeper zookeeper converter test [#30678](https://github.com/ClickHouse/ClickHouse/pull/30678) ([alesapin](https://github.com/alesapin)). +* Don't run build checks for documentation [#30681](https://github.com/ClickHouse/ClickHouse/pull/30681) ([alesapin](https://github.com/alesapin)). +* Refactoring of Log family [#30689](https://github.com/ClickHouse/ClickHouse/pull/30689) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update hardware page colors [#30719](https://github.com/ClickHouse/ClickHouse/pull/30719) ([Cody Baker](https://github.com/codyrobert)). +* Fix test_part_uuid::test_part_uuid_wal [#30723](https://github.com/ClickHouse/ClickHouse/pull/30723) ([Azat Khuzhin](https://github.com/azat)). +* Fix gtest_disk_encrypted (for new readFile/createReadBufferFromFileBase() interfaces) [#30725](https://github.com/ClickHouse/ClickHouse/pull/30725) ([Azat Khuzhin](https://github.com/azat)). +* tests/ci/docker_images_check: add missing time import [#30727](https://github.com/ClickHouse/ClickHouse/pull/30727) ([Azat Khuzhin](https://github.com/azat)). +* Add functional tests to github actions [#30729](https://github.com/ClickHouse/ClickHouse/pull/30729) ([alesapin](https://github.com/alesapin)). +* Run pylint over python scripts for github actions [#30733](https://github.com/ClickHouse/ClickHouse/pull/30733) ([Azat Khuzhin](https://github.com/azat)). +* DictionarySource unknown column name in dictionary fix [#30736](https://github.com/ClickHouse/ClickHouse/pull/30736) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01083_expressions_in_engine_arguments.sql [#30741](https://github.com/ClickHouse/ClickHouse/pull/30741) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Performance tests tuning [#30773](https://github.com/ClickHouse/ClickHouse/pull/30773) ([Azat Khuzhin](https://github.com/azat)). +* Remove cruft [#30780](https://github.com/ClickHouse/ClickHouse/pull/30780) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test 02022 [#30786](https://github.com/ClickHouse/ClickHouse/pull/30786) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Trying to make fuzzer check less hardcoded [#30795](https://github.com/ClickHouse/ClickHouse/pull/30795) ([alesapin](https://github.com/alesapin)). +* DictionarySource unknown column name fix [#30800](https://github.com/ClickHouse/ClickHouse/pull/30800) ([Maksim Kita](https://github.com/kitaisreal)). +* Add funding press release [#30806](https://github.com/ClickHouse/ClickHouse/pull/30806) ([Cody Baker](https://github.com/codyrobert)). +* Update installation success message [#30672](https://github.com/ClickHouse/ClickHouse/issues/30672) [#30830](https://github.com/ClickHouse/ClickHouse/pull/30830) ([Teja](https://github.com/tejasrivastav)). +* filelog engine tests improve [#30832](https://github.com/ClickHouse/ClickHouse/pull/30832) ([flynn](https://github.com/ucasfl)). +* Remove redundant from http buffer [#30837](https://github.com/ClickHouse/ClickHouse/pull/30837) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Docs check on github actions [#30839](https://github.com/ClickHouse/ClickHouse/pull/30839) ([alesapin](https://github.com/alesapin)). +* Fix fuzzer on master [#30841](https://github.com/ClickHouse/ClickHouse/pull/30841) ([alesapin](https://github.com/alesapin)). +* Disable check for ZooKeeper session uptime by default [#30847](https://github.com/ClickHouse/ClickHouse/pull/30847) ([Alexander Tokmakov](https://github.com/tavplubix)). +* enable modify table comment of rest table engine [#30852](https://github.com/ClickHouse/ClickHouse/pull/30852) ([flynn](https://github.com/ucasfl)). +* Fix typo [#30853](https://github.com/ClickHouse/ClickHouse/pull/30853) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed `--disable-net-host` in `runner` [#30863](https://github.com/ClickHouse/ClickHouse/pull/30863) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix protocol revision. [#30864](https://github.com/ClickHouse/ClickHouse/pull/30864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add docker cleanup after actions [#30868](https://github.com/ClickHouse/ClickHouse/pull/30868) ([alesapin](https://github.com/alesapin)). +* Fix error message in Keeper handler [#30880](https://github.com/ClickHouse/ClickHouse/pull/30880) ([alesapin](https://github.com/alesapin)). +* Better handling of `xtables.lock` in `runner` [#30892](https://github.com/ClickHouse/ClickHouse/pull/30892) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Reimplement [#28639](https://github.com/ClickHouse/ClickHouse/issues/28639) [#30903](https://github.com/ClickHouse/ClickHouse/pull/30903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Miscellaneous [#30917](https://github.com/ClickHouse/ClickHouse/pull/30917) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix yml in docs release [#30928](https://github.com/ClickHouse/ClickHouse/pull/30928) ([alesapin](https://github.com/alesapin)). +* Debug github event [#30929](https://github.com/ClickHouse/ClickHouse/pull/30929) ([alesapin](https://github.com/alesapin)). +* Fix docs release [#30933](https://github.com/ClickHouse/ClickHouse/pull/30933) ([alesapin](https://github.com/alesapin)). +* Fix style check [#30937](https://github.com/ClickHouse/ClickHouse/pull/30937) ([alesapin](https://github.com/alesapin)). +* Fix file progress for clickhouse-local [#30938](https://github.com/ClickHouse/ClickHouse/pull/30938) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flacky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix reading from TinyLog [#30941](https://github.com/ClickHouse/ClickHouse/pull/30941) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add github to known hosts in docs release [#30947](https://github.com/ClickHouse/ClickHouse/pull/30947) ([alesapin](https://github.com/alesapin)). +* Parse json from response in ci checks [#30948](https://github.com/ClickHouse/ClickHouse/pull/30948) ([alesapin](https://github.com/alesapin)). + #### Testing Improvement * Implemented structure-aware fuzzing approach in ClickHouse for select statement parser. [#30012](https://github.com/ClickHouse/ClickHouse/pull/30012) ([Paul](https://github.com/PaulCher)). diff --git a/docs/changelogs/v21.11.10.1-stable.md b/docs/changelogs/v21.11.10.1-stable.md index f07eccd30c5..c180918bb07 100644 --- a/docs/changelogs/v21.11.10.1-stable.md +++ b/docs/changelogs/v21.11.10.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.10.1-stable FIXME as compared to v21.11.9.1-stable diff --git a/docs/changelogs/v21.11.11.1-stable.md b/docs/changelogs/v21.11.11.1-stable.md index e46f43c53e0..922245d031e 100644 --- a/docs/changelogs/v21.11.11.1-stable.md +++ b/docs/changelogs/v21.11.11.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.11.2.2-stable.md b/docs/changelogs/v21.11.2.2-stable.md index f48f91a9b13..9a11dfd03c5 100644 --- a/docs/changelogs/v21.11.2.2-stable.md +++ b/docs/changelogs/v21.11.2.2-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,7 @@ * Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31026](https://github.com/ClickHouse/ClickHouse/issues/31026): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Return back accidentally removed code [#30996](https://github.com/ClickHouse/ClickHouse/pull/30996) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.11.3.6-stable.md b/docs/changelogs/v21.11.3.6-stable.md index bf46ecec590..5db65d42c6d 100644 --- a/docs/changelogs/v21.11.3.6-stable.md +++ b/docs/changelogs/v21.11.3.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.3.6-stable FIXME as compared to v21.11.2.2-stable #### Bug Fix diff --git a/docs/changelogs/v21.11.4.14-stable.md b/docs/changelogs/v21.11.4.14-stable.md index b3d44b8c193..be661ad244d 100644 --- a/docs/changelogs/v21.11.4.14-stable.md +++ b/docs/changelogs/v21.11.4.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.4.14-stable FIXME as compared to v21.11.3.6-stable #### Bug Fix @@ -15,3 +22,9 @@ * Backported in [#31283](https://github.com/ClickHouse/ClickHouse/issues/31283): Rename setting value `read_threadpool` to `threadpool` for setting `remote_filesystem_read_method`. [#31224](https://github.com/ClickHouse/ClickHouse/pull/31224) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix threadpool read for remote disks [#31112](https://github.com/ClickHouse/ClickHouse/pull/31112) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix check in async buffer [#31325](https://github.com/ClickHouse/ClickHouse/pull/31325) ([Kseniia Sumarokova](https://github.com/kssenii)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.11.5.33-stable.md b/docs/changelogs/v21.11.5.33-stable.md index 973c443d9f8..2895505715f 100644 --- a/docs/changelogs/v21.11.5.33-stable.md +++ b/docs/changelogs/v21.11.5.33-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.5.33-stable FIXME as compared to v21.11.4.14-stable #### Performance Improvement @@ -31,3 +38,7 @@ * Backported in [#32091](https://github.com/ClickHouse/ClickHouse/issues/32091): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#32020](https://github.com/ClickHouse/ClickHouse/issues/32020): Fix FileLog engine unnesessary create meta data directory when create table failed. Fix [#31962](https://github.com/ClickHouse/ClickHouse/issues/31962). [#31967](https://github.com/ClickHouse/ClickHouse/pull/31967) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.6.7-stable.md b/docs/changelogs/v21.11.6.7-stable.md index 1f3df589466..c3127723a68 100644 --- a/docs/changelogs/v21.11.6.7-stable.md +++ b/docs/changelogs/v21.11.6.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.6.7-stable FIXME as compared to v21.11.5.33-stable #### Bug Fix @@ -22,3 +29,8 @@ * NO CL ENTRY: 'Manual backport of [#31766](https://github.com/ClickHouse/ClickHouse/issues/31766) into 21.11'. [#32202](https://github.com/ClickHouse/ClickHouse/pull/32202) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.7.9-stable.md b/docs/changelogs/v21.11.7.9-stable.md index 1d907ad0ce1..5595e7bc0a9 100644 --- a/docs/changelogs/v21.11.7.9-stable.md +++ b/docs/changelogs/v21.11.7.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.7.9-stable FIXME as compared to v21.11.6.7-stable #### Bug Fix @@ -13,3 +20,9 @@ * Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.8.4-stable.md b/docs/changelogs/v21.11.8.4-stable.md index 0826b473758..c670180a52e 100644 --- a/docs/changelogs/v21.11.8.4-stable.md +++ b/docs/changelogs/v21.11.8.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -10,3 +17,8 @@ * Backported in [#32891](https://github.com/ClickHouse/ClickHouse/issues/32891): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#32792](https://github.com/ClickHouse/ClickHouse/issues/32792): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.9.1-stable.md b/docs/changelogs/v21.11.9.1-stable.md index c1754614c3c..bca99350b47 100644 --- a/docs/changelogs/v21.11.9.1-stable.md +++ b/docs/changelogs/v21.11.9.1-stable.md @@ -1,6 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md index f5e036c9c52..7ca0cbc3605 100644 --- a/docs/changelogs/v21.12.1.9017-prestable.md +++ b/docs/changelogs/v21.12.1.9017-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.1.9017-prestable FIXME as compared to v21.11.1.8636-prestable #### Backward Incompatible Change @@ -21,7 +28,7 @@ * Exposes all GlobalThreadPool configurations to the configuration files. [#31285](https://github.com/ClickHouse/ClickHouse/pull/31285) ([Tomáš Hromada](https://github.com/gyfis)). * Aliyun OSS Storage support. [#31286](https://github.com/ClickHouse/ClickHouse/pull/31286) ([cfcz48](https://github.com/cfcz48)). * Allow to print/parse names and types of colums in CustomSeparated input/output format. Add formats CustomSeparatedWithNames/WithNamesAndTypes similar to TSVWithNames/WithNamesAndTypes. [#31434](https://github.com/ClickHouse/ClickHouse/pull/31434) ([Kruglov Pavel](https://github.com/Avogar)). -* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([michael1589](https://github.com/michael1589)). +* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([Peng Liu](https://github.com/michael1589)). #### Performance Improvement * ... Allow to split GraphiteMergeTree rollup rules for plain/tagged metrics (optional rule_type field). [#25122](https://github.com/ClickHouse/ClickHouse/pull/25122) ([Michail Safronov](https://github.com/msaf1980)). @@ -54,7 +61,7 @@ * Add settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem` and `merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem`. [#30970](https://github.com/ClickHouse/ClickHouse/pull/30970) ([Kseniia Sumarokova](https://github.com/kssenii)). * Do not allow to drop a table or dictionary if some tables or dictionaries depend on it. [#30977](https://github.com/ClickHouse/ClickHouse/pull/30977) ([Alexander Tokmakov](https://github.com/tavplubix)). * Only grab AlterLock when we do alter command. Let's see if the assumption is correct. [#31010](https://github.com/ClickHouse/ClickHouse/pull/31010) ([Amos Bird](https://github.com/amosbird)). -* The local session inside a ClickHouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). +* The local session inside a Clickhouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). * Cancel vertical merges when partition is dropped. This is a follow-up of https://github.com/ClickHouse/ClickHouse/pull/25684 and https://github.com/ClickHouse/ClickHouse/pull/30996. [#31057](https://github.com/ClickHouse/ClickHouse/pull/31057) ([Amos Bird](https://github.com/amosbird)). * Support `IF EXISTS` modifier for `RENAME DATABASE`/`TABLE`/`DICTIONARY` query, If this directive is used, one will not get an error if the DATABASE/TABLE/DICTIONARY to be renamed doesn't exist. [#31081](https://github.com/ClickHouse/ClickHouse/pull/31081) ([victorgao](https://github.com/kafka1991)). * Function name normalization for ALTER queries. This helps avoid metadata mismatch between creating table with indices/projections and adding indices/projections via alter commands. This is a follow-up PR of https://github.com/ClickHouse/ClickHouse/pull/20174. Mark as improvements as there are no bug reports and the senario is somehow rare. [#31095](https://github.com/ClickHouse/ClickHouse/pull/31095) ([Amos Bird](https://github.com/amosbird)). @@ -204,3 +211,216 @@ * NO CL ENTRY: 'Revert "Add a test with 20000 mutations in one query"'. [#32326](https://github.com/ClickHouse/ClickHouse/pull/32326) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * NO CL ENTRY: 'Revert "Revert "Add a test with 20000 mutations in one query""'. [#32327](https://github.com/ClickHouse/ClickHouse/pull/32327) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Refactor pipeline executor [#19587](https://github.com/ClickHouse/ClickHouse/pull/19587) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support NULLS FIRST in KeyCondition [#29528](https://github.com/ClickHouse/ClickHouse/pull/29528) ([DimasKovas](https://github.com/DimasKovas)). +* Integration test improvements [#29806](https://github.com/ClickHouse/ClickHouse/pull/29806) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not allow zero-length reads [#30190](https://github.com/ClickHouse/ClickHouse/pull/30190) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_backward_compatibility [#30950](https://github.com/ClickHouse/ClickHouse/pull/30950) ([Ilya Yatsishin](https://github.com/qoega)). +* Add stress test to github actions [#30952](https://github.com/ClickHouse/ClickHouse/pull/30952) ([alesapin](https://github.com/alesapin)). +* Try smaller blacklist of non parallel integration tests [#30963](https://github.com/ClickHouse/ClickHouse/pull/30963) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flacky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Move access-rights source code [#30973](https://github.com/ClickHouse/ClickHouse/pull/30973) ([Vitaly Baranov](https://github.com/vitlibar)). +* Set output_format_avro_rows_in_file default to 1 [#30990](https://github.com/ClickHouse/ClickHouse/pull/30990) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove remaining usages of Y_IGNORE [#30993](https://github.com/ClickHouse/ClickHouse/pull/30993) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Return back accidentally removed code [#30996](https://github.com/ClickHouse/ClickHouse/pull/30996) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Rename AccessControlManager [#30998](https://github.com/ClickHouse/ClickHouse/pull/30998) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add fuzzer check to actions [#31002](https://github.com/ClickHouse/ClickHouse/pull/31002) ([alesapin](https://github.com/alesapin)). +* Small refactoring in formats [#31004](https://github.com/ClickHouse/ClickHouse/pull/31004) ([Kruglov Pavel](https://github.com/Avogar)). +* Changelog for version 21.11 [#31014](https://github.com/ClickHouse/ClickHouse/pull/31014) ([Rich Raposa](https://github.com/rfraposa)). +* perf: add missing DROP TABLE queries [#31028](https://github.com/ClickHouse/ClickHouse/pull/31028) ([Azat Khuzhin](https://github.com/azat)). +* Tune perf tests configs/scripts [#31029](https://github.com/ClickHouse/ClickHouse/pull/31029) ([Azat Khuzhin](https://github.com/azat)). +* Remove metric_log/part_log overrides in tests (enabled by default) [#31030](https://github.com/ClickHouse/ClickHouse/pull/31030) ([Azat Khuzhin](https://github.com/azat)). +* improve error message while using OFFSET FETCH clause without ORDER BY [#31031](https://github.com/ClickHouse/ClickHouse/pull/31031) ([SuperDJY](https://github.com/cmsxbc)). +* Log size of remapped memory (remap_executable) [#31033](https://github.com/ClickHouse/ClickHouse/pull/31033) ([Azat Khuzhin](https://github.com/azat)). +* Separate option for enabling fuse syntax for sum, avg, count [#31035](https://github.com/ClickHouse/ClickHouse/pull/31035) ([Vladimir C](https://github.com/vdimir)). +* Add integration test on top of github actions [#31045](https://github.com/ClickHouse/ClickHouse/pull/31045) ([alesapin](https://github.com/alesapin)). +* Fix intersecting parts in `parts_to_do` 2 [#31060](https://github.com/ClickHouse/ClickHouse/pull/31060) ([Alexander Tokmakov](https://github.com/tavplubix)). +* perf: switch *_log tables to Memory engine (attempt to reduce cache misses) [#31063](https://github.com/ClickHouse/ClickHouse/pull/31063) ([Azat Khuzhin](https://github.com/azat)). +* Add new employee photos to team [#31084](https://github.com/ClickHouse/ClickHouse/pull/31084) ([Cody Baker](https://github.com/codyrobert)). +* Update button widths for responsive sizing [#31085](https://github.com/ClickHouse/ClickHouse/pull/31085) ([Cody Baker](https://github.com/codyrobert)). +* Add overflow to benchmark tables [#31086](https://github.com/ClickHouse/ClickHouse/pull/31086) ([Cody Baker](https://github.com/codyrobert)). +* Remove padding below greenhouse iframe [#31087](https://github.com/ClickHouse/ClickHouse/pull/31087) ([Cody Baker](https://github.com/codyrobert)). +* Crb update case study cards [#31088](https://github.com/ClickHouse/ClickHouse/pull/31088) ([Cody Baker](https://github.com/codyrobert)). +* Fix threadpool read for remote disks [#31112](https://github.com/ClickHouse/ClickHouse/pull/31112) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Doc build: remove single.md [#31118](https://github.com/ClickHouse/ClickHouse/pull/31118) ([lehasm](https://github.com/lehasm)). +* remove unnecessary assert in StorageFileLog [#31119](https://github.com/ClickHouse/ClickHouse/pull/31119) ([nauta](https://github.com/nautaa)). +* Add lambda for approve [#31139](https://github.com/ClickHouse/ClickHouse/pull/31139) ([alesapin](https://github.com/alesapin)). +* Do not include unnecessary experimental/type_traits [#31147](https://github.com/ClickHouse/ClickHouse/pull/31147) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Disable optimize_syntax_fuse_functions by default [#31149](https://github.com/ClickHouse/ClickHouse/pull/31149) ([Vladimir C](https://github.com/vdimir)). +* Add pvs studio to actions [#31156](https://github.com/ClickHouse/ClickHouse/pull/31156) ([alesapin](https://github.com/alesapin)). +* Add cherry-pick on github actions [#31158](https://github.com/ClickHouse/ClickHouse/pull/31158) ([alesapin](https://github.com/alesapin)). +* correct disk space calculations [#31159](https://github.com/ClickHouse/ClickHouse/pull/31159) ([Alexandre Snarskii](https://github.com/snar)). +* Fix typo [#31164](https://github.com/ClickHouse/ClickHouse/pull/31164) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update text on thank you page [#31166](https://github.com/ClickHouse/ClickHouse/pull/31166) ([Cody Baker](https://github.com/codyrobert)). +* Add unit tests to CI [#31175](https://github.com/ClickHouse/ClickHouse/pull/31175) ([alesapin](https://github.com/alesapin)). +* Debug cherry-pick CI [#31177](https://github.com/ClickHouse/ClickHouse/pull/31177) ([alesapin](https://github.com/alesapin)). +* Update docker_compose_postgres.yml [#31179](https://github.com/ClickHouse/ClickHouse/pull/31179) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix integration tests [#31223](https://github.com/ClickHouse/ClickHouse/pull/31223) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Relax test 02026_storage_filelog_largefile.sh [#31225](https://github.com/ClickHouse/ClickHouse/pull/31225) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo in USE_MYSQL check [#31226](https://github.com/ClickHouse/ClickHouse/pull/31226) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Download ccache if not exists in builds [#31227](https://github.com/ClickHouse/ClickHouse/pull/31227) ([alesapin](https://github.com/alesapin)). +* Disable fuzzer builds in CI [#31244](https://github.com/ClickHouse/ClickHouse/pull/31244) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add flaky check to new CI [#31248](https://github.com/ClickHouse/ClickHouse/pull/31248) ([alesapin](https://github.com/alesapin)). +* Fix test [#31250](https://github.com/ClickHouse/ClickHouse/pull/31250) ([Kseniia Sumarokova](https://github.com/kssenii)). +* move InputCreatorFunc to InputCreator [#31258](https://github.com/ClickHouse/ClickHouse/pull/31258) ([Alex Cao](https://github.com/cccgp)). +* Print warning during old directories cleanup in MergeTree only if it is old [#31259](https://github.com/ClickHouse/ClickHouse/pull/31259) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup extern ProfileEvents/CurrentMetrics and add a style check [#31260](https://github.com/ClickHouse/ClickHouse/pull/31260) ([Azat Khuzhin](https://github.com/azat)). +* Fix and refactor WriteBiffer-s a little [#31265](https://github.com/ClickHouse/ClickHouse/pull/31265) ([Kruglov Pavel](https://github.com/Avogar)). +* Followup to ccache build [#31287](https://github.com/ClickHouse/ClickHouse/pull/31287) ([alesapin](https://github.com/alesapin)). +* Add compatibility check [#31294](https://github.com/ClickHouse/ClickHouse/pull/31294) ([alesapin](https://github.com/alesapin)). +* Add assertions to ZooKeeperLock [#31295](https://github.com/ClickHouse/ClickHouse/pull/31295) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add split build check [#31299](https://github.com/ClickHouse/ClickHouse/pull/31299) ([alesapin](https://github.com/alesapin)). +* Remove strange code from mutations [#31300](https://github.com/ClickHouse/ClickHouse/pull/31300) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable integration tests in new CI [#31301](https://github.com/ClickHouse/ClickHouse/pull/31301) ([alesapin](https://github.com/alesapin)). +* Merging [#31081](https://github.com/ClickHouse/ClickHouse/issues/31081) [#31305](https://github.com/ClickHouse/ClickHouse/pull/31305) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Debug rabbitmq tests [#31316](https://github.com/ClickHouse/ClickHouse/pull/31316) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible data-race in case of query cancellation with async_socket_for_remote [#31317](https://github.com/ClickHouse/ClickHouse/pull/31317) ([Azat Khuzhin](https://github.com/azat)). +* Improve fuzzer report in case of fuzzer killed [#31324](https://github.com/ClickHouse/ClickHouse/pull/31324) ([Azat Khuzhin](https://github.com/azat)). +* Fix check in async buffer [#31325](https://github.com/ClickHouse/ClickHouse/pull/31325) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add Blog Post for 21.11 Release Update [#31326](https://github.com/ClickHouse/ClickHouse/pull/31326) ([Cody Baker](https://github.com/codyrobert)). +* Add blog post for Moscow meetup [#31327](https://github.com/ClickHouse/ClickHouse/pull/31327) ([Cody Baker](https://github.com/codyrobert)). +* Do not try to resolve temporary tables from global context [#31333](https://github.com/ClickHouse/ClickHouse/pull/31333) ([Azat Khuzhin](https://github.com/azat)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove some copypaste from CI [#31340](https://github.com/ClickHouse/ClickHouse/pull/31340) ([alesapin](https://github.com/alesapin)). +* Fix test_kafka_insert_avro by pinning avro version [#31387](https://github.com/ClickHouse/ClickHouse/pull/31387) ([Azat Khuzhin](https://github.com/azat)). +* Fix QueryScope in MaterializedMySQLSyncThread [#31392](https://github.com/ClickHouse/ClickHouse/pull/31392) ([Azat Khuzhin](https://github.com/azat)). +* Check stderr is writable before reopining it (to avoid losing errors) [#31393](https://github.com/ClickHouse/ClickHouse/pull/31393) ([Azat Khuzhin](https://github.com/azat)). +* Remove thread_local std::string [#31400](https://github.com/ClickHouse/ClickHouse/pull/31400) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix client [#31403](https://github.com/ClickHouse/ClickHouse/pull/31403) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove excessive debug info from the log message in DDLWorker [#31406](https://github.com/ClickHouse/ClickHouse/pull/31406) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Turn on more CI checks [#31413](https://github.com/ClickHouse/ClickHouse/pull/31413) ([alesapin](https://github.com/alesapin)). +* Update description for webinar calendar links [#31433](https://github.com/ClickHouse/ClickHouse/pull/31433) ([Cody Baker](https://github.com/codyrobert)). +* Trying to debug integration tests [#31443](https://github.com/ClickHouse/ClickHouse/pull/31443) ([alesapin](https://github.com/alesapin)). +* Try increase `snapshot_distance` for functional tests [#31448](https://github.com/ClickHouse/ClickHouse/pull/31448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Minor improvement for test_replicated_fetches_bandwidth [#31451](https://github.com/ClickHouse/ClickHouse/pull/31451) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18787](https://github.com/ClickHouse/ClickHouse/issues/18787) (Constraints) [#31476](https://github.com/ClickHouse/ClickHouse/pull/31476) ([Anton Popov](https://github.com/CurtizJ)). +* Crb support page [#31490](https://github.com/ClickHouse/ClickHouse/pull/31490) ([Cody Baker](https://github.com/codyrobert)). +* Add new team members to company page [#31491](https://github.com/ClickHouse/ClickHouse/pull/31491) ([Cody Baker](https://github.com/codyrobert)). +* [ci] whitelist codyrobert [#31492](https://github.com/ClickHouse/ClickHouse/pull/31492) ([Ivan Blinkov](https://github.com/blinkov)). +* Reapply style changes to hardware page [#31506](https://github.com/ClickHouse/ClickHouse/pull/31506) ([Cody Baker](https://github.com/codyrobert)). +* Split row policy and quota headers [#31509](https://github.com/ClickHouse/ClickHouse/pull/31509) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not clean iptables rules in session-scope fixture [#31527](https://github.com/ClickHouse/ClickHouse/pull/31527) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Push tests results to CI database [#31540](https://github.com/ClickHouse/ClickHouse/pull/31540) ([alesapin](https://github.com/alesapin)). +* Remove strange multimap for mutations in StorageMergeTree [#31542](https://github.com/ClickHouse/ClickHouse/pull/31542) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove duplicated not compressed logs from CI (actions) [#31544](https://github.com/ClickHouse/ClickHouse/pull/31544) ([Azat Khuzhin](https://github.com/azat)). +* Fix 02010_lc_native flakiness (Query with id = 123456 is already running) [#31556](https://github.com/ClickHouse/ClickHouse/pull/31556) ([Azat Khuzhin](https://github.com/azat)). +* Make 01114_database_atomic more stable in debug builds [#31564](https://github.com/ClickHouse/ClickHouse/pull/31564) ([Azat Khuzhin](https://github.com/azat)). +* Fix MySQLWire format (this will also fix performance tests) [#31565](https://github.com/ClickHouse/ClickHouse/pull/31565) ([Azat Khuzhin](https://github.com/azat)). +* get Build ID via Section headers first [#31566](https://github.com/ClickHouse/ClickHouse/pull/31566) ([Ilya Golshtein](https://github.com/ilejn)). +* Try to debug expired sessions [#31584](https://github.com/ClickHouse/ClickHouse/pull/31584) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix cleanup in integration tests [#31605](https://github.com/ClickHouse/ClickHouse/pull/31605) ([Vitaly Baranov](https://github.com/vitlibar)). +* Stop all periodic reloading of all the configuration files on shutdown earlier [#31607](https://github.com/ClickHouse/ClickHouse/pull/31607) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix build requirements for unit tests [#31617](https://github.com/ClickHouse/ClickHouse/pull/31617) ([alesapin](https://github.com/alesapin)). +* Add workflow run for release branches [#31618](https://github.com/ClickHouse/ClickHouse/pull/31618) ([alesapin](https://github.com/alesapin)). +* Other event for release PR [#31619](https://github.com/ClickHouse/ClickHouse/pull/31619) ([alesapin](https://github.com/alesapin)). +* Trying push event again [#31623](https://github.com/ClickHouse/ClickHouse/pull/31623) ([alesapin](https://github.com/alesapin)). +* Add github actions for master [#31629](https://github.com/ClickHouse/ClickHouse/pull/31629) ([alesapin](https://github.com/alesapin)). +* Fix master yml [#31630](https://github.com/ClickHouse/ClickHouse/pull/31630) ([alesapin](https://github.com/alesapin)). +* fix kerberized_hadoop image [#31637](https://github.com/ClickHouse/ClickHouse/pull/31637) ([Constantine Peresypkin](https://github.com/pkit)). +* [ci] add flickerbox-tom to whitelist [#31651](https://github.com/ClickHouse/ClickHouse/pull/31651) ([Ivan Blinkov](https://github.com/blinkov)). +* Try to fix possible data race in RemoteQueryExecutorReadContext [#31652](https://github.com/ClickHouse/ClickHouse/pull/31652) ([Kruglov Pavel](https://github.com/Avogar)). +* Integration tests flaky check and small fixes [#31654](https://github.com/ClickHouse/ClickHouse/pull/31654) ([alesapin](https://github.com/alesapin)). +* Update base64 library [#31677](https://github.com/ClickHouse/ClickHouse/pull/31677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix typo [#31678](https://github.com/ClickHouse/ClickHouse/pull/31678) ([flynn](https://github.com/ucasfl)). +* Try fix OOMs with TSAN [#31685](https://github.com/ClickHouse/ClickHouse/pull/31685) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix workflow in master [#31688](https://github.com/ClickHouse/ClickHouse/pull/31688) ([alesapin](https://github.com/alesapin)). +* Add perf test for writing valid UTF8 [#31695](https://github.com/ClickHouse/ClickHouse/pull/31695) ([Kruglov Pavel](https://github.com/Avogar)). +* hdfs disable stderr logging [#31703](https://github.com/ClickHouse/ClickHouse/pull/31703) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible Logical error: Cannot write to finalized buffer [#31712](https://github.com/ClickHouse/ClickHouse/pull/31712) ([Kruglov Pavel](https://github.com/Avogar)). +* 02050: Use CLICKHOUSE_TMP and delete files when finished [#31713](https://github.com/ClickHouse/ClickHouse/pull/31713) ([Raúl Marín](https://github.com/Algunenano)). +* Make 02112_with_fill_interval independent of the server timezone [#31714](https://github.com/ClickHouse/ClickHouse/pull/31714) ([Raúl Marín](https://github.com/Algunenano)). +* 02010_lc_native: Generate a new id for each query [#31720](https://github.com/ClickHouse/ClickHouse/pull/31720) ([Raúl Marín](https://github.com/Algunenano)). +* 00623_replicated_truncate_table_zookeeper_long: Wait for truncate in replicas [#31721](https://github.com/ClickHouse/ClickHouse/pull/31721) ([Raúl Marín](https://github.com/Algunenano)). +* Try to push data into another ci database. [#31724](https://github.com/ClickHouse/ClickHouse/pull/31724) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove OpenCL completely [#31744](https://github.com/ClickHouse/ClickHouse/pull/31744) ([Timur Magomedov](https://github.com/tmagomedov)). +* Fix diff for backports. [#31765](https://github.com/ClickHouse/ClickHouse/pull/31765) ([alesapin](https://github.com/alesapin)). +* Fail fasttest, builds and functional checks if some tests was not successful [#31767](https://github.com/ClickHouse/ClickHouse/pull/31767) ([alesapin](https://github.com/alesapin)). +* Improve how queries are output in the performance dashboard [#31780](https://github.com/ClickHouse/ClickHouse/pull/31780) ([Raúl Marín](https://github.com/Algunenano)). +* Use version from git describe in builds [#31782](https://github.com/ClickHouse/ClickHouse/pull/31782) ([alesapin](https://github.com/alesapin)). +* Fix stylecheck for tests/ci/push_to_artifactory.py [#31798](https://github.com/ClickHouse/ClickHouse/pull/31798) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Shorter stress and fuzzer tests [#31803](https://github.com/ClickHouse/ClickHouse/pull/31803) ([alesapin](https://github.com/alesapin)). +* Fix oss-fuzz build [#31818](https://github.com/ClickHouse/ClickHouse/pull/31818) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix check in approve lambda [#31821](https://github.com/ClickHouse/ClickHouse/pull/31821) ([alesapin](https://github.com/alesapin)). +* Cover query_views_log [#31825](https://github.com/ClickHouse/ClickHouse/pull/31825) ([Azat Khuzhin](https://github.com/azat)). +* Forbid files that differ only by character case [#31834](https://github.com/ClickHouse/ClickHouse/pull/31834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added ubsan suppression for libprotobuf-mutator [#31835](https://github.com/ClickHouse/ClickHouse/pull/31835) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update 01155_rename_move_materialized_view.sql [#31849](https://github.com/ClickHouse/ClickHouse/pull/31849) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix style. [#31850](https://github.com/ClickHouse/ClickHouse/pull/31850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Minor improvements to DUMP macro [#31858](https://github.com/ClickHouse/ClickHouse/pull/31858) ([Vasily Nemkov](https://github.com/Enmk)). +* Get rid of build numbers and simplify builds paths in S3 [#31861](https://github.com/ClickHouse/ClickHouse/pull/31861) ([alesapin](https://github.com/alesapin)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added -no-sanitize=unsigned-integer-overflow build flag [#31881](https://github.com/ClickHouse/ClickHouse/pull/31881) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix typos [#31886](https://github.com/ClickHouse/ClickHouse/pull/31886) ([Anton Popov](https://github.com/CurtizJ)). +* Try to fix flacky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce the files that depend on parser headers [#31896](https://github.com/ClickHouse/ClickHouse/pull/31896) ([Raúl Marín](https://github.com/Algunenano)). +* Fix magic_enum for debug helpers (fixes build w/ USE_DEBUG_HELPERS) [#31922](https://github.com/ClickHouse/ClickHouse/pull/31922) ([Azat Khuzhin](https://github.com/azat)). +* Remove some trash from build [#31923](https://github.com/ClickHouse/ClickHouse/pull/31923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add json type to changes in documentation [#31926](https://github.com/ClickHouse/ClickHouse/pull/31926) ([alesapin](https://github.com/alesapin)). +* fix some broken links [#31948](https://github.com/ClickHouse/ClickHouse/pull/31948) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Kill container in integration tests if it's already running [#31950](https://github.com/ClickHouse/ClickHouse/pull/31950) ([alesapin](https://github.com/alesapin)). +* Drop libc-headers [#31951](https://github.com/ClickHouse/ClickHouse/pull/31951) ([Raúl Marín](https://github.com/Algunenano)). +* Give some love to macOS platform [#31957](https://github.com/ClickHouse/ClickHouse/pull/31957) ([Denis Glazachev](https://github.com/traceon)). +* Fix segfault in MaterializedMySQL [#31960](https://github.com/ClickHouse/ClickHouse/pull/31960) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix docs check [#31980](https://github.com/ClickHouse/ClickHouse/pull/31980) ([alesapin](https://github.com/alesapin)). +* Make stress tests slightly more stable [#31985](https://github.com/ClickHouse/ClickHouse/pull/31985) ([alesapin](https://github.com/alesapin)). +* Add rest functional tests to CI [#31987](https://github.com/ClickHouse/ClickHouse/pull/31987) ([alesapin](https://github.com/alesapin)). +* Add special builds to CI [#31991](https://github.com/ClickHouse/ClickHouse/pull/31991) ([alesapin](https://github.com/alesapin)). +* Run less tests for backport branches [#31992](https://github.com/ClickHouse/ClickHouse/pull/31992) ([alesapin](https://github.com/alesapin)). +* Fix race in ParallelFormattingOutputFormat constructor [#32004](https://github.com/ClickHouse/ClickHouse/pull/32004) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix build on master [#32009](https://github.com/ClickHouse/ClickHouse/pull/32009) ([alesapin](https://github.com/alesapin)). +* Fix style-check for ProfileEvents checking [#32033](https://github.com/ClickHouse/ClickHouse/pull/32033) ([Azat Khuzhin](https://github.com/azat)). +* Provide clickhouse binary w/o debug symbols (stripped) in fasttest [#32036](https://github.com/ClickHouse/ClickHouse/pull/32036) ([Azat Khuzhin](https://github.com/azat)). +* Minor fixes for `StorageMergeTree` [#32037](https://github.com/ClickHouse/ClickHouse/pull/32037) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Remove most of old checks [#32041](https://github.com/ClickHouse/ClickHouse/pull/32041) ([alesapin](https://github.com/alesapin)). +* Remove fast test from master [#32042](https://github.com/ClickHouse/ClickHouse/pull/32042) ([alesapin](https://github.com/alesapin)). +* Better scripts for runners [#32043](https://github.com/ClickHouse/ClickHouse/pull/32043) ([alesapin](https://github.com/alesapin)). +* Fix force tests label [#32044](https://github.com/ClickHouse/ClickHouse/pull/32044) ([alesapin](https://github.com/alesapin)). +* Don't run checks for label event [#32046](https://github.com/ClickHouse/ClickHouse/pull/32046) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 00925 [#32050](https://github.com/ClickHouse/ClickHouse/pull/32050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Cancel redundant checks with lambda [#32051](https://github.com/ClickHouse/ClickHouse/pull/32051) ([alesapin](https://github.com/alesapin)). +* Fix flaky integration test for MaterializedMySQL CREATE TABLE LIKE [#32052](https://github.com/ClickHouse/ClickHouse/pull/32052) ([Stig Bakken](https://github.com/stigsb)). +* Use functional test group for tests with thread sanitizer [#32062](https://github.com/ClickHouse/ClickHouse/pull/32062) ([alesapin](https://github.com/alesapin)). +* Add ability for lightweight checks rerun [#32064](https://github.com/ClickHouse/ClickHouse/pull/32064) ([alesapin](https://github.com/alesapin)). +* Increase length of random database in clickhouse-test [#32094](https://github.com/ClickHouse/ClickHouse/pull/32094) ([Azat Khuzhin](https://github.com/azat)). +* make looping in H3 funcs uniform [#32110](https://github.com/ClickHouse/ClickHouse/pull/32110) ([Bharat Nallan](https://github.com/bharatnc)). +* Remove PVS check from master [#32114](https://github.com/ClickHouse/ClickHouse/pull/32114) ([alesapin](https://github.com/alesapin)). +* Fix flaky keeper whitelist test [#32115](https://github.com/ClickHouse/ClickHouse/pull/32115) ([alesapin](https://github.com/alesapin)). +* Fix flacky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Move fuzzers and unit tests to another group [#32120](https://github.com/ClickHouse/ClickHouse/pull/32120) ([alesapin](https://github.com/alesapin)). +* Add a test with 20000 mutations in one query [#32122](https://github.com/ClickHouse/ClickHouse/pull/32122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Change test 02117_custom_separated_with_names_and_types [#32123](https://github.com/ClickHouse/ClickHouse/pull/32123) ([Kruglov Pavel](https://github.com/Avogar)). +* Use seq_cst semantic for MergeTreeBackgroundExecutor mertic. [#32125](https://github.com/ClickHouse/ClickHouse/pull/32125) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove 02116_global_in_time_limit. [#32126](https://github.com/ClickHouse/ClickHouse/pull/32126) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fixing postgres tests [#32129](https://github.com/ClickHouse/ClickHouse/pull/32129) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Small improvements in lambda code [#32155](https://github.com/ClickHouse/ClickHouse/pull/32155) ([alesapin](https://github.com/alesapin)). +* Update featured image for 21.11 release blog post [#32156](https://github.com/ClickHouse/ClickHouse/pull/32156) ([Cody Baker](https://github.com/codyrobert)). +* Fix CI [#32159](https://github.com/ClickHouse/ClickHouse/pull/32159) ([alesapin](https://github.com/alesapin)). +* tests/ci: do not compress logs twice [#32162](https://github.com/ClickHouse/ClickHouse/pull/32162) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: do not use random generator with shared state [#32163](https://github.com/ClickHouse/ClickHouse/pull/32163) ([Azat Khuzhin](https://github.com/azat)). +* Fix stress tests [#32164](https://github.com/ClickHouse/ClickHouse/pull/32164) ([Azat Khuzhin](https://github.com/azat)). +* Fix QueryProfiler (query_profiler_{cpu,real}_time_period_ns) reset [#32165](https://github.com/ClickHouse/ClickHouse/pull/32165) ([Azat Khuzhin](https://github.com/azat)). +* MaterializedMySQL support VARBINARY type [#32173](https://github.com/ClickHouse/ClickHouse/pull/32173) ([zzsmdfj](https://github.com/zzsmdfj)). +* perf: fix waiting of the server after running tests [#32174](https://github.com/ClickHouse/ClickHouse/pull/32174) ([Azat Khuzhin](https://github.com/azat)). +* Better output for some actions [#32175](https://github.com/ClickHouse/ClickHouse/pull/32175) ([alesapin](https://github.com/alesapin)). +* Use ccache in fast test [#32177](https://github.com/ClickHouse/ClickHouse/pull/32177) ([alesapin](https://github.com/alesapin)). +* Fix window view tests [#32178](https://github.com/ClickHouse/ClickHouse/pull/32178) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Function accurateCastOrDefault remove separate branch [#32184](https://github.com/ClickHouse/ClickHouse/pull/32184) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for [#32186](https://github.com/ClickHouse/ClickHouse/issues/32186) [#32203](https://github.com/ClickHouse/ClickHouse/pull/32203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix uncaught exception in DatabaseLazy [#32206](https://github.com/ClickHouse/ClickHouse/pull/32206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update ASTCreateQuery.cpp [#32208](https://github.com/ClickHouse/ClickHouse/pull/32208) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flacky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix jemalloc under osx [#32219](https://github.com/ClickHouse/ClickHouse/pull/32219) ([Azat Khuzhin](https://github.com/azat)). +* Add missing timezones to some tests [#32222](https://github.com/ClickHouse/ClickHouse/pull/32222) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix versioning of aggregate functions (fixes performance tests) [#32236](https://github.com/ClickHouse/ClickHouse/pull/32236) ([Azat Khuzhin](https://github.com/azat)). +* Disable window view tests temporarily because still flacky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo in tupleToNameValuePairs doc [#32262](https://github.com/ClickHouse/ClickHouse/pull/32262) ([Vladimir C](https://github.com/vdimir)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible crash in DataTypeAggregateFunction [#32287](https://github.com/ClickHouse/ClickHouse/pull/32287) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update backport.py [#32323](https://github.com/ClickHouse/ClickHouse/pull/32323) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix graphite-bench build [#32351](https://github.com/ClickHouse/ClickHouse/pull/32351) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Revert "graphite: split tagged/plain rollup rules (for merges perfoma… [#32376](https://github.com/ClickHouse/ClickHouse/pull/32376) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Another attempt to fix unit test Executor::RemoveTasksStress [#32390](https://github.com/ClickHouse/ClickHouse/pull/32390) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.12.2.17-stable.md b/docs/changelogs/v21.12.2.17-stable.md index 909bc7917c7..a35006d9c57 100644 --- a/docs/changelogs/v21.12.2.17-stable.md +++ b/docs/changelogs/v21.12.2.17-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.2.17-stable FIXME as compared to v21.12.1.9017-prestable #### Bug Fix @@ -20,3 +27,11 @@ * Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32389](https://github.com/ClickHouse/ClickHouse/pull/32389) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.12.3.32-stable.md b/docs/changelogs/v21.12.3.32-stable.md index b650f62dd34..f14057981a2 100644 --- a/docs/changelogs/v21.12.3.32-stable.md +++ b/docs/changelogs/v21.12.3.32-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.3.32-stable FIXME as compared to v21.12.2.17-stable #### Bug Fix @@ -15,3 +22,12 @@ * Backported in [#33100](https://github.com/ClickHouse/ClickHouse/issues/33100): Fix Context leak in case of cancel_http_readonly_queries_on_client_close (i.e. leaking of external tables that had been uploaded the the server and other resources). [#32982](https://github.com/ClickHouse/ClickHouse/pull/32982) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#33123](https://github.com/ClickHouse/ClickHouse/issues/33123): Fix error `Invalid version for SerializationLowCardinality key column` in case of reading from `LowCardinality` column with `local_filesystem_read_prefetch` or `remote_filesystem_read_prefetch` enabled. [#33046](https://github.com/ClickHouse/ClickHouse/pull/33046) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve quota's end-of-interval calculations. [#32575](https://github.com/ClickHouse/ClickHouse/pull/32575) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix race in skipping index of type `hypothesis` [#32756](https://github.com/ClickHouse/ClickHouse/pull/32756) ([Anton Popov](https://github.com/CurtizJ)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.12.4.1-stable.md b/docs/changelogs/v21.12.4.1-stable.md index 7c028592876..d08997378af 100644 --- a/docs/changelogs/v21.12.4.1-stable.md +++ b/docs/changelogs/v21.12.4.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.4.1-stable FIXME as compared to v21.12.3.32-stable #### Improvement diff --git a/docs/changelogs/v21.2.1.5869-prestable.md b/docs/changelogs/v21.2.1.5869-prestable.md index 43703bc13b9..ae5efdcf14f 100644 --- a/docs/changelogs/v21.2.1.5869-prestable.md +++ b/docs/changelogs/v21.2.1.5869-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.1.5869-prestable FIXME as compared to v21.1.1.5646-prestable #### Backward Incompatible Change @@ -154,3 +161,116 @@ * NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). * NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add ANTLR tests check [#18624](https://github.com/ClickHouse/ClickHouse/pull/18624) ([Ivan](https://github.com/abyss7)). +* Add more debug info to PollingQueue exception. [#18922](https://github.com/ClickHouse/ClickHouse/pull/18922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More checks in merge tree writer wide [#18928](https://github.com/ClickHouse/ClickHouse/pull/18928) ([alesapin](https://github.com/alesapin)). +* Try to remove ActionsDAG::removeColumn [#18953](https://github.com/ClickHouse/ClickHouse/pull/18953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rarely flaky test 01459_manual_write_to_replicas_quorum [#18970](https://github.com/ClickHouse/ClickHouse/pull/18970) ([alesapin](https://github.com/alesapin)). +* fix some wrong words in comment [#18998](https://github.com/ClickHouse/ClickHouse/pull/18998) ([flynn](https://github.com/ucasfl)). +* Add tests from [#15889](https://github.com/ClickHouse/ClickHouse/issues/15889) [#19007](https://github.com/ClickHouse/ClickHouse/pull/19007) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix manipulators for common string types [#19011](https://github.com/ClickHouse/ClickHouse/pull/19011) ([alesapin](https://github.com/alesapin)). +* Fix misleading error message while inserting in a table function [#19013](https://github.com/ClickHouse/ClickHouse/pull/19013) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicate UUIDs of LiveView on server startup [#19020](https://github.com/ClickHouse/ClickHouse/pull/19020) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [wip] WINDOW clause [#19022](https://github.com/ClickHouse/ClickHouse/pull/19022) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Avoid redundant exception while dropping part [#19040](https://github.com/ClickHouse/ClickHouse/pull/19040) ([alesapin](https://github.com/alesapin)). +* More debug for stateless tests writer [#19055](https://github.com/ClickHouse/ClickHouse/pull/19055) ([alesapin](https://github.com/alesapin)). +* Update test containers [#19058](https://github.com/ClickHouse/ClickHouse/pull/19058) ([filimonov](https://github.com/filimonov)). +* Support operations with views in ANTLR parser [#19063](https://github.com/ClickHouse/ClickHouse/pull/19063) ([Anton Popov](https://github.com/CurtizJ)). +* Run more tests in SQLancer [#19077](https://github.com/ClickHouse/ClickHouse/pull/19077) ([Ilya Yatsishin](https://github.com/qoega)). +* Update Cassandra driver library [#19091](https://github.com/ClickHouse/ClickHouse/pull/19091) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add metrics for part number in MergeTree in ClickHouse [#19122](https://github.com/ClickHouse/ClickHouse/pull/19122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#19136](https://github.com/ClickHouse/ClickHouse/pull/19136) ([Amos Bird](https://github.com/amosbird)). +* [ANTLR] Print errors through LOG_ERROR [#19137](https://github.com/ClickHouse/ClickHouse/pull/19137) ([Ivan](https://github.com/abyss7)). +* MemoryTracker: Do not ignore server memory limits during blocking by default [#19146](https://github.com/ClickHouse/ClickHouse/pull/19146) ([Azat Khuzhin](https://github.com/azat)). +* speed up some perf tests (for other machines) [#19154](https://github.com/ClickHouse/ClickHouse/pull/19154) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Cover distributed send with different headers [#19155](https://github.com/ClickHouse/ClickHouse/pull/19155) ([Azat Khuzhin](https://github.com/azat)). +* Add missing type check in dictHas for COMPLEX_KEY_SSD_CACHE layout [#19164](https://github.com/ClickHouse/ClickHouse/pull/19164) ([Azat Khuzhin](https://github.com/azat)). +* test for [#18839](https://github.com/ClickHouse/ClickHouse/issues/18839) Expand_macros_for_fetchPartition [#19200](https://github.com/ClickHouse/ClickHouse/pull/19200) ([Denny Crane](https://github.com/den-crane)). +* add MySQL Var check [#19205](https://github.com/ClickHouse/ClickHouse/pull/19205) ([TCeason](https://github.com/TCeason)). +* Tiny changes in DistributedBlockOutputStream [#19206](https://github.com/ClickHouse/ClickHouse/pull/19206) ([Azat Khuzhin](https://github.com/azat)). +* curl dependency tiny fixes [#19210](https://github.com/ClickHouse/ClickHouse/pull/19210) ([Azat Khuzhin](https://github.com/azat)). +* Fix MSan error in rocksdb [#19213](https://github.com/ClickHouse/ClickHouse/issues/19213) [#19214](https://github.com/ClickHouse/ClickHouse/pull/19214) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix MSan report in Kerberos library [#19215](https://github.com/ClickHouse/ClickHouse/pull/19215) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make symbolizers available in fuzzer Docker image [#19220](https://github.com/ClickHouse/ClickHouse/pull/19220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add redundant test for Yandex banner system [#19235](https://github.com/ClickHouse/ClickHouse/pull/19235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `-- { echo }` hint preserve leading comments [#19236](https://github.com/ClickHouse/ClickHouse/pull/19236) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Disable msan for epoll array in libuv [#19244](https://github.com/ClickHouse/ClickHouse/pull/19244) ([alesapin](https://github.com/alesapin)). +* Remove tsan supression [#19250](https://github.com/ClickHouse/ClickHouse/pull/19250) ([alesapin](https://github.com/alesapin)). +* Consolidate the test hint handling [#19254](https://github.com/ClickHouse/ClickHouse/pull/19254) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Support 'keys' and 'values' subcolumns in type Map [#19273](https://github.com/ClickHouse/ClickHouse/pull/19273) ([Anton Popov](https://github.com/CurtizJ)). +* Split TestKeeperStorage and processing thread [#19284](https://github.com/ClickHouse/ClickHouse/pull/19284) ([alesapin](https://github.com/alesapin)). +* move ctr from private to delete [#19285](https://github.com/ClickHouse/ClickHouse/pull/19285) ([flynn](https://github.com/ucasfl)). +* Avoid mixing output from parallel test runs [#19298](https://github.com/ClickHouse/ClickHouse/pull/19298) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed race between copy-constructor and addQueryAccessInfo [#19313](https://github.com/ClickHouse/ClickHouse/pull/19313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add Sanitizer report issue template [#19318](https://github.com/ClickHouse/ClickHouse/pull/19318) ([Ilya Yatsishin](https://github.com/qoega)). +* fix sleep with infinite input [#19343](https://github.com/ClickHouse/ClickHouse/pull/19343) ([flynn](https://github.com/ucasfl)). +* ISSUES-18684 fix MaterializeMySQL integration test failure [#19344](https://github.com/ClickHouse/ClickHouse/pull/19344) ([Winter Zhang](https://github.com/zhang2014)). +* Fix race condition in TestKeeperHandler on session finish [#19355](https://github.com/ClickHouse/ClickHouse/pull/19355) ([alesapin](https://github.com/alesapin)). +* Fix several cases, while reading subcolumns [#19358](https://github.com/ClickHouse/ClickHouse/pull/19358) ([Anton Popov](https://github.com/CurtizJ)). +* Reconnect after client errors [#19361](https://github.com/ClickHouse/ClickHouse/pull/19361) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SQLancer binary changed its name [#19363](https://github.com/ClickHouse/ClickHouse/pull/19363) ([Ilya Yatsishin](https://github.com/qoega)). +* Better logging in MySQLHandler [#19365](https://github.com/ClickHouse/ClickHouse/pull/19365) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix IDisk::open parameters to match posix open [#19372](https://github.com/ClickHouse/ClickHouse/pull/19372) ([Alexander Gololobov](https://github.com/davenger)). +* MacOS fixed build issues [#19377](https://github.com/ClickHouse/ClickHouse/pull/19377) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log message with elapsed time while pushing to view [#19378](https://github.com/ClickHouse/ClickHouse/pull/19378) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in aggregate function sum [#19385](https://github.com/ClickHouse/ClickHouse/pull/19385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast in incorrect usage of extractAllGroups [#19393](https://github.com/ClickHouse/ClickHouse/pull/19393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileExactWeighted [#19394](https://github.com/ClickHouse/ClickHouse/pull/19394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Limit max memory usage in fuzz testing [#19396](https://github.com/ClickHouse/ClickHouse/pull/19396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check literal types in RewriteSumIfFunctionMatcher [#19406](https://github.com/ClickHouse/ClickHouse/pull/19406) ([Vladimir C](https://github.com/vdimir)). +* mirror changes in code and comment [#19410](https://github.com/ClickHouse/ClickHouse/pull/19410) ([flynn](https://github.com/ucasfl)). +* Fix one more race in TestKeeper [#19412](https://github.com/ClickHouse/ClickHouse/pull/19412) ([alesapin](https://github.com/alesapin)). +* Continue fix for Block structure mismatch in PipelineExecuting stream [#19414](https://github.com/ClickHouse/ClickHouse/pull/19414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Integration tests: print stderr of failed subprocess [#19431](https://github.com/ClickHouse/ClickHouse/pull/19431) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't dwell on the past in query fuzzer [#19442](https://github.com/ClickHouse/ClickHouse/pull/19442) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove outdated suppressions [#19444](https://github.com/ClickHouse/ClickHouse/pull/19444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build with unbundled poco [#19450](https://github.com/ClickHouse/ClickHouse/pull/19450) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in arraySlice and substring [#19459](https://github.com/ClickHouse/ClickHouse/pull/19459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in runningDifference [#19460](https://github.com/ClickHouse/ClickHouse/pull/19460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid UBSan report in greatCircleDistance [#19461](https://github.com/ClickHouse/ClickHouse/pull/19461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* UBsan report fixes (arraySlice, addMinutes/addHours/addWeeks/addDays, sumWithOverflow(Decimal)) [#19466](https://github.com/ClickHouse/ClickHouse/pull/19466) ([Azat Khuzhin](https://github.com/azat)). +* Remove complications from FunctionsAES [#19467](https://github.com/ClickHouse/ClickHouse/pull/19467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insufficient args check in AES functions [#19474](https://github.com/ClickHouse/ClickHouse/pull/19474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in avgWeighted [#19475](https://github.com/ClickHouse/ClickHouse/pull/19475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove part of misleading exception message [#19487](https://github.com/ClickHouse/ClickHouse/pull/19487) ([flynn](https://github.com/ucasfl)). +* Bug fix : support const column processing in mapContains, mapKeys, mapValues functions [#19515](https://github.com/ClickHouse/ClickHouse/pull/19515) ([hexiaoting](https://github.com/hexiaoting)). +* More diagnostics in fuzzer [#19524](https://github.com/ClickHouse/ClickHouse/pull/19524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01459_manual_write_to_replicas flaky test [#19551](https://github.com/ClickHouse/ClickHouse/pull/19551) ([alesapin](https://github.com/alesapin)). +* Check for hung queries or server hung in fast test [#19558](https://github.com/ClickHouse/ClickHouse/pull/19558) ([alesapin](https://github.com/alesapin)). +* DateLUTImpl::addYears(...): suppress UBSan [#19566](https://github.com/ClickHouse/ClickHouse/pull/19566) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18549](https://github.com/ClickHouse/ClickHouse/issues/18549) [#19583](https://github.com/ClickHouse/ClickHouse/pull/19583) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra submodule with patch by @leshikus [#19590](https://github.com/ClickHouse/ClickHouse/pull/19590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused include header [#19598](https://github.com/ClickHouse/ClickHouse/pull/19598) ([BohuTANG](https://github.com/BohuTANG)). +* Fix merge join constants [#19648](https://github.com/ClickHouse/ClickHouse/pull/19648) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better abstractions in disk interface [#19660](https://github.com/ClickHouse/ClickHouse/pull/19660) ([Anton Popov](https://github.com/CurtizJ)). +* Fix deadlock in testkeeper [#19661](https://github.com/ClickHouse/ClickHouse/pull/19661) ([alesapin](https://github.com/alesapin)). +* fix special build on clang11 [#19663](https://github.com/ClickHouse/ClickHouse/pull/19663) ([flynn](https://github.com/ucasfl)). +* Require current_database filter for tests with query_log/query_thread_log [#19675](https://github.com/ClickHouse/ClickHouse/pull/19675) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test tiny cleanup [#19676](https://github.com/ClickHouse/ClickHouse/pull/19676) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash after merging ActionsDAG. [#19704](https://github.com/ClickHouse/ClickHouse/pull/19704) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test concat_nary_const_with_nonconst_segfault [#19711](https://github.com/ClickHouse/ClickHouse/pull/19711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Run tests in parallel in flaky check [#19715](https://github.com/ClickHouse/ClickHouse/pull/19715) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix 01115_join_with_dictionary flakiness [#19723](https://github.com/ClickHouse/ClickHouse/pull/19723) ([Azat Khuzhin](https://github.com/azat)). +* add empty line after error messages in client [#19724](https://github.com/ClickHouse/ClickHouse/pull/19724) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update QueryPlan tree optimization traverse. [#19725](https://github.com/ClickHouse/ClickHouse/pull/19725) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add missing lsof for fasttest docker image [#19751](https://github.com/ClickHouse/ClickHouse/pull/19751) ([Azat Khuzhin](https://github.com/azat)). +* Make Fuzzer more reliable [#19752](https://github.com/ClickHouse/ClickHouse/pull/19752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code improvement in JOIN [#19758](https://github.com/ClickHouse/ClickHouse/pull/19758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make integration odbc tests idempotent [#19759](https://github.com/ClickHouse/ClickHouse/pull/19759) ([alesapin](https://github.com/alesapin)). +* Small fixes for fasttest [#19760](https://github.com/ClickHouse/ClickHouse/pull/19760) ([alesapin](https://github.com/alesapin)). +* LowCardinality UUID fix [#19767](https://github.com/ClickHouse/ClickHouse/pull/19767) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log comment when running .sh tests [#19774](https://github.com/ClickHouse/ClickHouse/pull/19774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in "round" [#19786](https://github.com/ClickHouse/ClickHouse/pull/19786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test script for [#19794](https://github.com/ClickHouse/ClickHouse/issues/19794) [#19798](https://github.com/ClickHouse/ClickHouse/pull/19798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix dependency on ODBC for Yandex internal build [#19804](https://github.com/ClickHouse/ClickHouse/pull/19804) ([Alexander Gololobov](https://github.com/davenger)). +* Don't run all stateless tests in parallel [#19806](https://github.com/ClickHouse/ClickHouse/pull/19806) ([alesapin](https://github.com/alesapin)). +* Avoid losing exception messages in logs under high memory pressure [#19824](https://github.com/ClickHouse/ClickHouse/pull/19824) ([Azat Khuzhin](https://github.com/azat)). +* Try to make test_dir.tar smaller [#19833](https://github.com/ClickHouse/ClickHouse/pull/19833) ([filimonov](https://github.com/filimonov)). +* style-check tiny fixes [#19834](https://github.com/ClickHouse/ClickHouse/pull/19834) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in DateTimeAddInterval [#19859](https://github.com/ClickHouse/ClickHouse/pull/19859) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix debug assertion in Hyperscan [#19860](https://github.com/ClickHouse/ClickHouse/pull/19860) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in GatherUtils [#19862](https://github.com/ClickHouse/ClickHouse/pull/19862) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.10.48-stable.md b/docs/changelogs/v21.2.10.48-stable.md index 11eea960931..f9d29374e91 100644 --- a/docs/changelogs/v21.2.10.48-stable.md +++ b/docs/changelogs/v21.2.10.48-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.10.48-stable FIXME as compared to v21.2.9.41-stable #### Improvement @@ -9,3 +16,7 @@ * Backported in [#23032](https://github.com/ClickHouse/ClickHouse/issues/23032): Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#23171](https://github.com/ClickHouse/ClickHouse/issues/23171): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.2.2.8-stable.md b/docs/changelogs/v21.2.2.8-stable.md index 73baea91547..d3b1f43bcc3 100644 --- a/docs/changelogs/v21.2.2.8-stable.md +++ b/docs/changelogs/v21.2.2.8-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.2.8-stable FIXME as compared to v21.1.1.5646-prestable #### Backward Incompatible Change @@ -68,7 +75,7 @@ * Add separate pool for message brokers (RabbitMQ and Kafka). [#19722](https://github.com/ClickHouse/ClickHouse/pull/19722) ([Azat Khuzhin](https://github.com/azat)). * In distributed queries if the setting `async_socket_for_remote` is enabled, it was possible to get stack overflow at least in debug build configuration if very deeply nested data type is used in table (e.g. `Array(Array(Array(...more...)))`). This fixes [#19108](https://github.com/ClickHouse/ClickHouse/issues/19108). This change introduces minor backward incompatibility: excessive parenthesis in type definitions no longer supported, example: `Array((UInt8))`. [#19736](https://github.com/ClickHouse/ClickHouse/pull/19736) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Table function `S3` will use global region if the region can't be determined exactly. This closes [#10998](https://github.com/ClickHouse/ClickHouse/issues/10998). [#19750](https://github.com/ClickHouse/ClickHouse/pull/19750) ([Vladimir Chebotarev](https://github.com/excitoon)). -* ClickHouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). +* Clickhouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). * Correctly output infinite arguments for `formatReadableTimeDelta` function. In previous versions, there was implicit conversion to implementation specific integer value. [#19791](https://github.com/ClickHouse/ClickHouse/pull/19791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * `S3` table function now supports `auto` compression mode (autodetect). This closes [#18754](https://github.com/ClickHouse/ClickHouse/issues/18754). [#19793](https://github.com/ClickHouse/ClickHouse/pull/19793) ([Vladimir Chebotarev](https://github.com/excitoon)). * Set charset to utf8mb4 when interacting with remote MySQL servers. Fixes [#19795](https://github.com/ClickHouse/ClickHouse/issues/19795). [#19800](https://github.com/ClickHouse/ClickHouse/pull/19800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -164,3 +171,117 @@ * NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). * NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add ANTLR tests check [#18624](https://github.com/ClickHouse/ClickHouse/pull/18624) ([Ivan](https://github.com/abyss7)). +* Add more debug info to PollingQueue exception. [#18922](https://github.com/ClickHouse/ClickHouse/pull/18922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More checks in merge tree writer wide [#18928](https://github.com/ClickHouse/ClickHouse/pull/18928) ([alesapin](https://github.com/alesapin)). +* Try to remove ActionsDAG::removeColumn [#18953](https://github.com/ClickHouse/ClickHouse/pull/18953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rarely flaky test 01459_manual_write_to_replicas_quorum [#18970](https://github.com/ClickHouse/ClickHouse/pull/18970) ([alesapin](https://github.com/alesapin)). +* fix some wrong words in comment [#18998](https://github.com/ClickHouse/ClickHouse/pull/18998) ([flynn](https://github.com/ucasfl)). +* Add tests from [#15889](https://github.com/ClickHouse/ClickHouse/issues/15889) [#19007](https://github.com/ClickHouse/ClickHouse/pull/19007) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix manipulators for common string types [#19011](https://github.com/ClickHouse/ClickHouse/pull/19011) ([alesapin](https://github.com/alesapin)). +* Fix misleading error message while inserting in a table function [#19013](https://github.com/ClickHouse/ClickHouse/pull/19013) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicate UUIDs of LiveView on server startup [#19020](https://github.com/ClickHouse/ClickHouse/pull/19020) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [wip] WINDOW clause [#19022](https://github.com/ClickHouse/ClickHouse/pull/19022) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Avoid redundant exception while dropping part [#19040](https://github.com/ClickHouse/ClickHouse/pull/19040) ([alesapin](https://github.com/alesapin)). +* More debug for stateless tests writer [#19055](https://github.com/ClickHouse/ClickHouse/pull/19055) ([alesapin](https://github.com/alesapin)). +* Update test containers [#19058](https://github.com/ClickHouse/ClickHouse/pull/19058) ([filimonov](https://github.com/filimonov)). +* Support operations with views in ANTLR parser [#19063](https://github.com/ClickHouse/ClickHouse/pull/19063) ([Anton Popov](https://github.com/CurtizJ)). +* Run more tests in SQLancer [#19077](https://github.com/ClickHouse/ClickHouse/pull/19077) ([Ilya Yatsishin](https://github.com/qoega)). +* Update Cassandra driver library [#19091](https://github.com/ClickHouse/ClickHouse/pull/19091) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add metrics for part number in MergeTree in ClickHouse [#19122](https://github.com/ClickHouse/ClickHouse/pull/19122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#19136](https://github.com/ClickHouse/ClickHouse/pull/19136) ([Amos Bird](https://github.com/amosbird)). +* [ANTLR] Print errors through LOG_ERROR [#19137](https://github.com/ClickHouse/ClickHouse/pull/19137) ([Ivan](https://github.com/abyss7)). +* MemoryTracker: Do not ignore server memory limits during blocking by default [#19146](https://github.com/ClickHouse/ClickHouse/pull/19146) ([Azat Khuzhin](https://github.com/azat)). +* speed up some perf tests (for other machines) [#19154](https://github.com/ClickHouse/ClickHouse/pull/19154) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Cover distributed send with different headers [#19155](https://github.com/ClickHouse/ClickHouse/pull/19155) ([Azat Khuzhin](https://github.com/azat)). +* Add missing type check in dictHas for COMPLEX_KEY_SSD_CACHE layout [#19164](https://github.com/ClickHouse/ClickHouse/pull/19164) ([Azat Khuzhin](https://github.com/azat)). +* test for [#18839](https://github.com/ClickHouse/ClickHouse/issues/18839) Expand_macros_for_fetchPartition [#19200](https://github.com/ClickHouse/ClickHouse/pull/19200) ([Denny Crane](https://github.com/den-crane)). +* add MySQL Var check [#19205](https://github.com/ClickHouse/ClickHouse/pull/19205) ([TCeason](https://github.com/TCeason)). +* Tiny changes in DistributedBlockOutputStream [#19206](https://github.com/ClickHouse/ClickHouse/pull/19206) ([Azat Khuzhin](https://github.com/azat)). +* curl dependency tiny fixes [#19210](https://github.com/ClickHouse/ClickHouse/pull/19210) ([Azat Khuzhin](https://github.com/azat)). +* Fix MSan error in rocksdb [#19213](https://github.com/ClickHouse/ClickHouse/issues/19213) [#19214](https://github.com/ClickHouse/ClickHouse/pull/19214) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix MSan report in Kerberos library [#19215](https://github.com/ClickHouse/ClickHouse/pull/19215) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make symbolizers available in fuzzer Docker image [#19220](https://github.com/ClickHouse/ClickHouse/pull/19220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add redundant test for Yandex banner system [#19235](https://github.com/ClickHouse/ClickHouse/pull/19235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `-- { echo }` hint preserve leading comments [#19236](https://github.com/ClickHouse/ClickHouse/pull/19236) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Disable msan for epoll array in libuv [#19244](https://github.com/ClickHouse/ClickHouse/pull/19244) ([alesapin](https://github.com/alesapin)). +* Remove tsan supression [#19250](https://github.com/ClickHouse/ClickHouse/pull/19250) ([alesapin](https://github.com/alesapin)). +* Consolidate the test hint handling [#19254](https://github.com/ClickHouse/ClickHouse/pull/19254) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Support 'keys' and 'values' subcolumns in type Map [#19273](https://github.com/ClickHouse/ClickHouse/pull/19273) ([Anton Popov](https://github.com/CurtizJ)). +* Split TestKeeperStorage and processing thread [#19284](https://github.com/ClickHouse/ClickHouse/pull/19284) ([alesapin](https://github.com/alesapin)). +* move ctr from private to delete [#19285](https://github.com/ClickHouse/ClickHouse/pull/19285) ([flynn](https://github.com/ucasfl)). +* Avoid mixing output from parallel test runs [#19298](https://github.com/ClickHouse/ClickHouse/pull/19298) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed race between copy-constructor and addQueryAccessInfo [#19313](https://github.com/ClickHouse/ClickHouse/pull/19313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add Sanitizer report issue template [#19318](https://github.com/ClickHouse/ClickHouse/pull/19318) ([Ilya Yatsishin](https://github.com/qoega)). +* fix sleep with infinite input [#19343](https://github.com/ClickHouse/ClickHouse/pull/19343) ([flynn](https://github.com/ucasfl)). +* ISSUES-18684 fix MaterializeMySQL integration test failure [#19344](https://github.com/ClickHouse/ClickHouse/pull/19344) ([Winter Zhang](https://github.com/zhang2014)). +* Fix race condition in TestKeeperHandler on session finish [#19355](https://github.com/ClickHouse/ClickHouse/pull/19355) ([alesapin](https://github.com/alesapin)). +* Fix several cases, while reading subcolumns [#19358](https://github.com/ClickHouse/ClickHouse/pull/19358) ([Anton Popov](https://github.com/CurtizJ)). +* Reconnect after client errors [#19361](https://github.com/ClickHouse/ClickHouse/pull/19361) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SQLancer binary changed its name [#19363](https://github.com/ClickHouse/ClickHouse/pull/19363) ([Ilya Yatsishin](https://github.com/qoega)). +* Better logging in MySQLHandler [#19365](https://github.com/ClickHouse/ClickHouse/pull/19365) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix IDisk::open parameters to match posix open [#19372](https://github.com/ClickHouse/ClickHouse/pull/19372) ([Alexander Gololobov](https://github.com/davenger)). +* MacOS fixed build issues [#19377](https://github.com/ClickHouse/ClickHouse/pull/19377) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log message with elapsed time while pushing to view [#19378](https://github.com/ClickHouse/ClickHouse/pull/19378) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in aggregate function sum [#19385](https://github.com/ClickHouse/ClickHouse/pull/19385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast in incorrect usage of extractAllGroups [#19393](https://github.com/ClickHouse/ClickHouse/pull/19393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileExactWeighted [#19394](https://github.com/ClickHouse/ClickHouse/pull/19394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Limit max memory usage in fuzz testing [#19396](https://github.com/ClickHouse/ClickHouse/pull/19396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check literal types in RewriteSumIfFunctionMatcher [#19406](https://github.com/ClickHouse/ClickHouse/pull/19406) ([Vladimir C](https://github.com/vdimir)). +* mirror changes in code and comment [#19410](https://github.com/ClickHouse/ClickHouse/pull/19410) ([flynn](https://github.com/ucasfl)). +* Fix one more race in TestKeeper [#19412](https://github.com/ClickHouse/ClickHouse/pull/19412) ([alesapin](https://github.com/alesapin)). +* Continue fix for Block structure mismatch in PipelineExecuting stream [#19414](https://github.com/ClickHouse/ClickHouse/pull/19414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Integration tests: print stderr of failed subprocess [#19431](https://github.com/ClickHouse/ClickHouse/pull/19431) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't dwell on the past in query fuzzer [#19442](https://github.com/ClickHouse/ClickHouse/pull/19442) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove outdated suppressions [#19444](https://github.com/ClickHouse/ClickHouse/pull/19444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build with unbundled poco [#19450](https://github.com/ClickHouse/ClickHouse/pull/19450) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in arraySlice and substring [#19459](https://github.com/ClickHouse/ClickHouse/pull/19459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in runningDifference [#19460](https://github.com/ClickHouse/ClickHouse/pull/19460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid UBSan report in greatCircleDistance [#19461](https://github.com/ClickHouse/ClickHouse/pull/19461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* UBsan report fixes (arraySlice, addMinutes/addHours/addWeeks/addDays, sumWithOverflow(Decimal)) [#19466](https://github.com/ClickHouse/ClickHouse/pull/19466) ([Azat Khuzhin](https://github.com/azat)). +* Remove complications from FunctionsAES [#19467](https://github.com/ClickHouse/ClickHouse/pull/19467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insufficient args check in AES functions [#19474](https://github.com/ClickHouse/ClickHouse/pull/19474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in avgWeighted [#19475](https://github.com/ClickHouse/ClickHouse/pull/19475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove part of misleading exception message [#19487](https://github.com/ClickHouse/ClickHouse/pull/19487) ([flynn](https://github.com/ucasfl)). +* Bug fix : support const column processing in mapContains, mapKeys, mapValues functions [#19515](https://github.com/ClickHouse/ClickHouse/pull/19515) ([hexiaoting](https://github.com/hexiaoting)). +* More diagnostics in fuzzer [#19524](https://github.com/ClickHouse/ClickHouse/pull/19524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01459_manual_write_to_replicas flaky test [#19551](https://github.com/ClickHouse/ClickHouse/pull/19551) ([alesapin](https://github.com/alesapin)). +* Check for hung queries or server hung in fast test [#19558](https://github.com/ClickHouse/ClickHouse/pull/19558) ([alesapin](https://github.com/alesapin)). +* DateLUTImpl::addYears(...): suppress UBSan [#19566](https://github.com/ClickHouse/ClickHouse/pull/19566) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18549](https://github.com/ClickHouse/ClickHouse/issues/18549) [#19583](https://github.com/ClickHouse/ClickHouse/pull/19583) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra submodule with patch by @leshikus [#19590](https://github.com/ClickHouse/ClickHouse/pull/19590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused include header [#19598](https://github.com/ClickHouse/ClickHouse/pull/19598) ([BohuTANG](https://github.com/BohuTANG)). +* Fix merge join constants [#19648](https://github.com/ClickHouse/ClickHouse/pull/19648) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better abstractions in disk interface [#19660](https://github.com/ClickHouse/ClickHouse/pull/19660) ([Anton Popov](https://github.com/CurtizJ)). +* Fix deadlock in testkeeper [#19661](https://github.com/ClickHouse/ClickHouse/pull/19661) ([alesapin](https://github.com/alesapin)). +* fix special build on clang11 [#19663](https://github.com/ClickHouse/ClickHouse/pull/19663) ([flynn](https://github.com/ucasfl)). +* Require current_database filter for tests with query_log/query_thread_log [#19675](https://github.com/ClickHouse/ClickHouse/pull/19675) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test tiny cleanup [#19676](https://github.com/ClickHouse/ClickHouse/pull/19676) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash after merging ActionsDAG. [#19704](https://github.com/ClickHouse/ClickHouse/pull/19704) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test concat_nary_const_with_nonconst_segfault [#19711](https://github.com/ClickHouse/ClickHouse/pull/19711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Run tests in parallel in flaky check [#19715](https://github.com/ClickHouse/ClickHouse/pull/19715) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix 01115_join_with_dictionary flakiness [#19723](https://github.com/ClickHouse/ClickHouse/pull/19723) ([Azat Khuzhin](https://github.com/azat)). +* add empty line after error messages in client [#19724](https://github.com/ClickHouse/ClickHouse/pull/19724) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update QueryPlan tree optimization traverse. [#19725](https://github.com/ClickHouse/ClickHouse/pull/19725) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add missing lsof for fasttest docker image [#19751](https://github.com/ClickHouse/ClickHouse/pull/19751) ([Azat Khuzhin](https://github.com/azat)). +* Make Fuzzer more reliable [#19752](https://github.com/ClickHouse/ClickHouse/pull/19752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code improvement in JOIN [#19758](https://github.com/ClickHouse/ClickHouse/pull/19758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make integration odbc tests idempotent [#19759](https://github.com/ClickHouse/ClickHouse/pull/19759) ([alesapin](https://github.com/alesapin)). +* Small fixes for fasttest [#19760](https://github.com/ClickHouse/ClickHouse/pull/19760) ([alesapin](https://github.com/alesapin)). +* LowCardinality UUID fix [#19767](https://github.com/ClickHouse/ClickHouse/pull/19767) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log comment when running .sh tests [#19774](https://github.com/ClickHouse/ClickHouse/pull/19774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in "round" [#19786](https://github.com/ClickHouse/ClickHouse/pull/19786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test script for [#19794](https://github.com/ClickHouse/ClickHouse/issues/19794) [#19798](https://github.com/ClickHouse/ClickHouse/pull/19798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix dependency on ODBC for Yandex internal build [#19804](https://github.com/ClickHouse/ClickHouse/pull/19804) ([Alexander Gololobov](https://github.com/davenger)). +* Don't run all stateless tests in parallel [#19806](https://github.com/ClickHouse/ClickHouse/pull/19806) ([alesapin](https://github.com/alesapin)). +* Avoid losing exception messages in logs under high memory pressure [#19824](https://github.com/ClickHouse/ClickHouse/pull/19824) ([Azat Khuzhin](https://github.com/azat)). +* Try to make test_dir.tar smaller [#19833](https://github.com/ClickHouse/ClickHouse/pull/19833) ([filimonov](https://github.com/filimonov)). +* style-check tiny fixes [#19834](https://github.com/ClickHouse/ClickHouse/pull/19834) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in DateTimeAddInterval [#19859](https://github.com/ClickHouse/ClickHouse/pull/19859) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix debug assertion in Hyperscan [#19860](https://github.com/ClickHouse/ClickHouse/pull/19860) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in GatherUtils [#19862](https://github.com/ClickHouse/ClickHouse/pull/19862) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.2.3.15-stable.md b/docs/changelogs/v21.2.3.15-stable.md index 26653e780bb..0471c499748 100644 --- a/docs/changelogs/v21.2.3.15-stable.md +++ b/docs/changelogs/v21.2.3.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.3.15-stable FIXME as compared to v21.2.2.8-stable #### Bug Fix @@ -17,3 +24,7 @@ * NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.2: Fix access control manager destruction order"'. [#20397](https://github.com/ClickHouse/ClickHouse/pull/20397) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.2.4.6-stable.md b/docs/changelogs/v21.2.4.6-stable.md index 1605186701d..709abbcf730 100644 --- a/docs/changelogs/v21.2.4.6-stable.md +++ b/docs/changelogs/v21.2.4.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.4.6-stable FIXME as compared to v21.2.3.15-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.5.5-stable.md b/docs/changelogs/v21.2.5.5-stable.md index b5275e89519..c4b16d8123e 100644 --- a/docs/changelogs/v21.2.5.5-stable.md +++ b/docs/changelogs/v21.2.5.5-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.5.5-stable FIXME as compared to v21.2.4.6-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.6.1-stable.md b/docs/changelogs/v21.2.6.1-stable.md index 1f28c14c485..ef7b13e260e 100644 --- a/docs/changelogs/v21.2.6.1-stable.md +++ b/docs/changelogs/v21.2.6.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.6.1-stable FIXME as compared to v21.2.5.5-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.7.11-stable.md b/docs/changelogs/v21.2.7.11-stable.md index 1f0f94ee0bf..277ddb9fcb3 100644 --- a/docs/changelogs/v21.2.7.11-stable.md +++ b/docs/changelogs/v21.2.7.11-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.7.11-stable FIXME as compared to v21.2.6.1-stable #### Bug Fix @@ -10,3 +17,7 @@ * Backported in [#21979](https://github.com/ClickHouse/ClickHouse/issues/21979): Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#22140](https://github.com/ClickHouse/ClickHouse/issues/22140): The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.2.8.31-stable.md b/docs/changelogs/v21.2.8.31-stable.md index 884dcb5a649..d7f7e5a7099 100644 --- a/docs/changelogs/v21.2.8.31-stable.md +++ b/docs/changelogs/v21.2.8.31-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.8.31-stable FIXME as compared to v21.2.7.11-stable #### Bug Fix @@ -28,3 +35,7 @@ * Backported in [#22699](https://github.com/ClickHouse/ClickHouse/issues/22699): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). * Backported in [#22740](https://github.com/ClickHouse/ClickHouse/issues/22740): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Mannual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.2 [#22505](https://github.com/ClickHouse/ClickHouse/pull/22505) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.2.9.41-stable.md b/docs/changelogs/v21.2.9.41-stable.md index ab4303aaa2a..d3a8644a879 100644 --- a/docs/changelogs/v21.2.9.41-stable.md +++ b/docs/changelogs/v21.2.9.41-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.9.41-stable FIXME as compared to v21.2.8.31-stable #### Improvement @@ -15,3 +22,8 @@ #### Build/Testing/Packaging Improvement * Backported in [#22814](https://github.com/ClickHouse/ClickHouse/issues/22814): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.1.6185-prestable.md b/docs/changelogs/v21.3.1.6185-prestable.md index dabfe3cfeb3..8d2a222f8aa 100644 --- a/docs/changelogs/v21.3.1.6185-prestable.md +++ b/docs/changelogs/v21.3.1.6185-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.1.6185-prestable FIXME as compared to v21.2.1.5869-prestable #### Backward Incompatible Change @@ -157,3 +164,162 @@ * NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Pytest check [#18972](https://github.com/ClickHouse/ClickHouse/pull/18972) ([Ivan](https://github.com/abyss7)). +* [wip] support RANGE frame for window functions [#19299](https://github.com/ClickHouse/ClickHouse/pull/19299) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* In memory coordination inside ClickHouse [#19580](https://github.com/ClickHouse/ClickHouse/pull/19580) ([alesapin](https://github.com/alesapin)). +* client: more suggestions [#19584](https://github.com/ClickHouse/ClickHouse/pull/19584) ([Azat Khuzhin](https://github.com/azat)). +* Allow to run all style checks in one file [#19726](https://github.com/ClickHouse/ClickHouse/pull/19726) ([Anton Popov](https://github.com/CurtizJ)). +* Fix [#19371](https://github.com/ClickHouse/ClickHouse/issues/19371) [#19765](https://github.com/ClickHouse/ClickHouse/pull/19765) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor code improvements around ThreadStatus [#19771](https://github.com/ClickHouse/ClickHouse/pull/19771) ([Alexander Tokmakov](https://github.com/tavplubix)). +* continue of [#19487](https://github.com/ClickHouse/ClickHouse/issues/19487) [#19801](https://github.com/ClickHouse/ClickHouse/pull/19801) ([flynn](https://github.com/ucasfl)). +* Fix UBSan report in intDiv [#19876](https://github.com/ClickHouse/ClickHouse/pull/19876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow writing partial buffer [#19886](https://github.com/ClickHouse/ClickHouse/pull/19886) ([Azat Khuzhin](https://github.com/azat)). +* Remove an always-false condition from query parser [#19919](https://github.com/ClickHouse/ClickHouse/pull/19919) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UNBOUNDED FOLLOWING frame end [#19921](https://github.com/ClickHouse/ClickHouse/pull/19921) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix logical error in INSERT VALUES [#19925](https://github.com/ClickHouse/ClickHouse/pull/19925) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix test keeper integration tests [#19942](https://github.com/ClickHouse/ClickHouse/pull/19942) ([alesapin](https://github.com/alesapin)). +* Fix build [#19948](https://github.com/ClickHouse/ClickHouse/pull/19948) ([Kruglov Pavel](https://github.com/Avogar)). +* CURRENT ROW and offset for start of ROWS frame [#19951](https://github.com/ClickHouse/ClickHouse/pull/19951) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in geoHashesInBox [#19956](https://github.com/ClickHouse/ClickHouse/pull/19956) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MSan annotation for system.stack_trace [#19957](https://github.com/ClickHouse/ClickHouse/pull/19957) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration tests [#19971](https://github.com/ClickHouse/ClickHouse/pull/19971) ([Azat Khuzhin](https://github.com/azat)). +* Do not use inputs which values are known constants in ActionsDAG. [#19991](https://github.com/ClickHouse/ClickHouse/pull/19991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* update perf tests [#20000](https://github.com/ClickHouse/ClickHouse/pull/20000) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UBsan fixes [#20011](https://github.com/ClickHouse/ClickHouse/pull/20011) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#20029](https://github.com/ClickHouse/ClickHouse/pull/20029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update changelog [#20033](https://github.com/ClickHouse/ClickHouse/pull/20033) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Close input ports in DelayedPortsProcessor as soon as all outputs are finished [#20035](https://github.com/ClickHouse/ClickHouse/pull/20035) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ReadBuffer: check for unread data on next() [#20037](https://github.com/ClickHouse/ClickHouse/pull/20037) ([Ivan](https://github.com/abyss7)). +* ReadBuffer: check that buffer position is not set beyond end [#20039](https://github.com/ClickHouse/ClickHouse/pull/20039) ([Ivan](https://github.com/abyss7)). +* CURRENT ROW frame start for RANGE frame [#20041](https://github.com/ClickHouse/ClickHouse/pull/20041) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix log message with elapsed time while pushing to views [#20047](https://github.com/ClickHouse/ClickHouse/pull/20047) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in pointInPolygon [#20049](https://github.com/ClickHouse/ClickHouse/pull/20049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests *.reference files [#20050](https://github.com/ClickHouse/ClickHouse/pull/20050) ([Azat Khuzhin](https://github.com/azat)). +* ROWS OFFSET frame end [#20060](https://github.com/ClickHouse/ClickHouse/pull/20060) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add more tests for modulo of division of negative number [#20063](https://github.com/ClickHouse/ClickHouse/pull/20063) ([hexiaoting](https://github.com/hexiaoting)). +* detect unmarked long tests in flaky check [#20068](https://github.com/ClickHouse/ClickHouse/pull/20068) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix removing of filter column when split filter actions. [#20073](https://github.com/ClickHouse/ClickHouse/pull/20073) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove AddingConstColumn step and transform. [#20077](https://github.com/ClickHouse/ClickHouse/pull/20077) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* LimitReadBuffer: check that position always advances [#20078](https://github.com/ClickHouse/ClickHouse/pull/20078) ([Ivan](https://github.com/abyss7)). +* Add fuzzer for ColumnsDescription [#20094](https://github.com/ClickHouse/ClickHouse/pull/20094) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build actions dag to evaluate missing defaults. [#20097](https://github.com/ClickHouse/ClickHouse/pull/20097) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove never existing insert_in_memory_parts_timeout [#20099](https://github.com/ClickHouse/ClickHouse/pull/20099) ([Azat Khuzhin](https://github.com/azat)). +* RANGE OFFSET window frame [#20111](https://github.com/ClickHouse/ClickHouse/pull/20111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to drop readonly tables [#20114](https://github.com/ClickHouse/ClickHouse/pull/20114) ([nvartolomei](https://github.com/nvartolomei)). +* Add logging if Poco cannot allocate thread in tcp server [#20120](https://github.com/ClickHouse/ClickHouse/pull/20120) ([alesapin](https://github.com/alesapin)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Useless changes [#20150](https://github.com/ClickHouse/ClickHouse/pull/20150) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove some useless code [#20152](https://github.com/ClickHouse/ClickHouse/pull/20152) ([flynn](https://github.com/ucasfl)). +* Allow using MergeTreeWhereOptimizer not only with MergeTree-based storages [#20153](https://github.com/ClickHouse/ClickHouse/pull/20153) ([Max Akhmedov](https://github.com/zlobober)). +* Fix build of utils [#20155](https://github.com/ClickHouse/ClickHouse/pull/20155) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in arrayCumSum [#20160](https://github.com/ClickHouse/ClickHouse/pull/20160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.2 [#20185](https://github.com/ClickHouse/ClickHouse/pull/20185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ubsan error [#20204](https://github.com/ClickHouse/ClickHouse/pull/20204) ([Amos Bird](https://github.com/amosbird)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving [#20206](https://github.com/ClickHouse/ClickHouse/pull/20206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove dead code [#20212](https://github.com/ClickHouse/ClickHouse/pull/20212) ([nvartolomei](https://github.com/nvartolomei)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). +* Fix benign race in system.parts [#20226](https://github.com/ClickHouse/ClickHouse/pull/20226) ([alesapin](https://github.com/alesapin)). +* Add final to some classes [#20228](https://github.com/ClickHouse/ClickHouse/pull/20228) ([alesapin](https://github.com/alesapin)). +* refine code in MergeTreeData::loadDataParts to avoid parsing WAL file as data part [#20231](https://github.com/ClickHouse/ClickHouse/pull/20231) ([Fuwang Hu](https://github.com/fuwhu)). +* fix a problem in ArithmeticOperationsInAgrFuncOptimize [#20246](https://github.com/ClickHouse/ClickHouse/pull/20246) ([flynn](https://github.com/ucasfl)). +* Add retries to test_access_control_on_cluster [#20247](https://github.com/ClickHouse/ClickHouse/pull/20247) ([Ilya Yatsishin](https://github.com/qoega)). +* Stable sort for test cases in test_dictionaries_all_layouts_separate_sources [#20248](https://github.com/ClickHouse/ClickHouse/pull/20248) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix not closed ports in DelayedPortsProcessor [#20251](https://github.com/ClickHouse/ClickHouse/pull/20251) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* print changed settings in fuzzer when the server dies [#20281](https://github.com/ClickHouse/ClickHouse/pull/20281) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Put windows in such an order that we can sort less [#20284](https://github.com/ClickHouse/ClickHouse/pull/20284) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Aggregate function deltaSum use restrict keyword [#20287](https://github.com/ClickHouse/ClickHouse/pull/20287) ([Maksim Kita](https://github.com/kitaisreal)). +* make window functions faster [#20293](https://github.com/ClickHouse/ClickHouse/pull/20293) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Accept arbitrary numeric types for numbers() arguments (for scientific notation) [#20301](https://github.com/ClickHouse/ClickHouse/pull/20301) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00738_lock_for_inner_table flakiness [#20305](https://github.com/ClickHouse/ClickHouse/pull/20305) ([Azat Khuzhin](https://github.com/azat)). +* Filter push down [#20341](https://github.com/ClickHouse/ClickHouse/pull/20341) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Common types template instantiations [#20348](https://github.com/ClickHouse/ClickHouse/pull/20348) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for sign column drop [#20398](https://github.com/ClickHouse/ClickHouse/pull/20398) ([alesapin](https://github.com/alesapin)). +* Merging [#19204](https://github.com/ClickHouse/ClickHouse/issues/19204) [#20399](https://github.com/ClickHouse/ClickHouse/pull/20399) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix 01700_system_zookeeper_path_in [#20449](https://github.com/ClickHouse/ClickHouse/pull/20449) ([alesapin](https://github.com/alesapin)). +* Print stack trace on SIGTRAP [#20453](https://github.com/ClickHouse/ClickHouse/pull/20453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in arrayDifference [#20458](https://github.com/ClickHouse/ClickHouse/pull/20458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress UBSan report in Decimal comparison [#20459](https://github.com/ClickHouse/ClickHouse/pull/20459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve backtrace catching on server failures in CI for stress tests [#20462](https://github.com/ClickHouse/ClickHouse/pull/20462) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server terminations due to write failures [#20465](https://github.com/ClickHouse/ClickHouse/pull/20465) ([Azat Khuzhin](https://github.com/azat)). +* Improve logging during reading from MergeTree [#20466](https://github.com/ClickHouse/ClickHouse/pull/20466) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in intDiv [#20475](https://github.com/ClickHouse/ClickHouse/pull/20475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* (unrealistic case, never possible in prod) HashTable fix potential bug during resize with nonstandard grower [#20489](https://github.com/ClickHouse/ClickHouse/pull/20489) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8654](https://github.com/ClickHouse/ClickHouse/issues/8654) [#20492](https://github.com/ClickHouse/ClickHouse/pull/20492) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#10893](https://github.com/ClickHouse/ClickHouse/issues/10893) [#20493](https://github.com/ClickHouse/ClickHouse/pull/20493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable testkeeper snapshots for tests [#20496](https://github.com/ClickHouse/ClickHouse/pull/20496) ([alesapin](https://github.com/alesapin)). +* Fix non-zero session reconnect in integration test [#20501](https://github.com/ClickHouse/ClickHouse/pull/20501) ([alesapin](https://github.com/alesapin)). +* Reduce test scale [#20511](https://github.com/ClickHouse/ClickHouse/pull/20511) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Quick build fix [#20513](https://github.com/ClickHouse/ClickHouse/pull/20513) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace null fields in tuple during parsing with default values [#20541](https://github.com/ClickHouse/ClickHouse/pull/20541) ([Maksim Kita](https://github.com/kitaisreal)). +* rabbitmq: add missing format factory settings [#20545](https://github.com/ClickHouse/ClickHouse/pull/20545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Persistent coordination log storage [#20585](https://github.com/ClickHouse/ClickHouse/pull/20585) ([alesapin](https://github.com/alesapin)). +* Add hung check to stress test [#20588](https://github.com/ClickHouse/ClickHouse/pull/20588) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ignore data store files [#20598](https://github.com/ClickHouse/ClickHouse/pull/20598) ([tison](https://github.com/tisonkun)). +* Dictionary create source with functions crash fix [#20623](https://github.com/ClickHouse/ClickHouse/pull/20623) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix fasttest retry for failed tests [#20627](https://github.com/ClickHouse/ClickHouse/pull/20627) ([alesapin](https://github.com/alesapin)). +* Don't backport base commit of branch in the same branch [#20628](https://github.com/ClickHouse/ClickHouse/pull/20628) ([Ivan](https://github.com/abyss7)). +* Add test for already fixed odbc Postgres date type conversion [#20712](https://github.com/ClickHouse/ClickHouse/pull/20712) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve backtrace catching on server failures in CI for fast-tests [#20864](https://github.com/ClickHouse/ClickHouse/pull/20864) ([Azat Khuzhin](https://github.com/azat)). +* Better postgres db engine numeric conversion [#20874](https://github.com/ClickHouse/ClickHouse/pull/20874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix undefined-behavior in ReservoirSamplerDeterministic.h [#20879](https://github.com/ClickHouse/ClickHouse/pull/20879) ([Vitaly Baranov](https://github.com/vitlibar)). +* ccache 4.2+ does not requires any quirks for SOURCE_DATE_EPOCH [#20883](https://github.com/ClickHouse/ClickHouse/pull/20883) ([Azat Khuzhin](https://github.com/azat)). +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix push to dockerhub [#20960](https://github.com/ClickHouse/ClickHouse/pull/20960) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better tests for protobuf format. [#20968](https://github.com/ClickHouse/ClickHouse/pull/20968) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function sumMap decimal fix [#20970](https://github.com/ClickHouse/ClickHouse/pull/20970) ([Maksim Kita](https://github.com/kitaisreal)). +* test for decimal ( p , s) in dictionaries [#20980](https://github.com/ClickHouse/ClickHouse/pull/20980) ([Denny Crane](https://github.com/den-crane)). +* Fix uncaught exception when HTTP client goes away [#20981](https://github.com/ClickHouse/ClickHouse/pull/20981) ([Azat Khuzhin](https://github.com/azat)). +* Increase buffer for uncaught exception / std::terminate [#20989](https://github.com/ClickHouse/ClickHouse/pull/20989) ([Azat Khuzhin](https://github.com/azat)). +* Constraints complex types support [#20990](https://github.com/ClickHouse/ClickHouse/pull/20990) ([Maksim Kita](https://github.com/kitaisreal)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving 2 [#20995](https://github.com/ClickHouse/ClickHouse/pull/20995) ([Amos Bird](https://github.com/amosbird)). +* Add log message when stacktrace cannot be obtained for thread [#20996](https://github.com/ClickHouse/ClickHouse/pull/20996) ([Azat Khuzhin](https://github.com/azat)). +* Preserve mysql logs in test_materialize_mysql_database [#21016](https://github.com/ClickHouse/ClickHouse/pull/21016) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes in Decimal [#21017](https://github.com/ClickHouse/ClickHouse/pull/21017) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* yamllint style check [#21019](https://github.com/ClickHouse/ClickHouse/pull/21019) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer size for trace collection [#21020](https://github.com/ClickHouse/ClickHouse/pull/21020) ([Azat Khuzhin](https://github.com/azat)). +* Try fix MaterializeMySQL integration test [#21021](https://github.com/ClickHouse/ClickHouse/pull/21021) ([Winter Zhang](https://github.com/zhang2014)). +* Fix performance tests (by avoid sharing status file for right and left server) [#21022](https://github.com/ClickHouse/ClickHouse/pull/21022) ([Azat Khuzhin](https://github.com/azat)). +* Revert "optimize aggfunc column data copy ([#19407](https://github.com/ClickHouse/ClickHouse/issues/19407))" [#21024](https://github.com/ClickHouse/ClickHouse/pull/21024) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Decimal arithmetic [#21025](https://github.com/ClickHouse/ClickHouse/pull/21025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update lowcardinality.md [#21033](https://github.com/ClickHouse/ClickHouse/pull/21033) ([Michael Monashev](https://github.com/MichaelMonashev)). +* Fix DateTime64 from Float [#21050](https://github.com/ClickHouse/ClickHouse/pull/21050) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#19376](https://github.com/ClickHouse/ClickHouse/issues/19376) [#21051](https://github.com/ClickHouse/ClickHouse/pull/21051) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#20620](https://github.com/ClickHouse/ClickHouse/issues/20620) [#21052](https://github.com/ClickHouse/ClickHouse/pull/21052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9781](https://github.com/ClickHouse/ClickHouse/issues/9781) [#21074](https://github.com/ClickHouse/ClickHouse/pull/21074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix various issues in DDLWorker (SIGSEGV and others) [#21079](https://github.com/ClickHouse/ClickHouse/pull/21079) ([Azat Khuzhin](https://github.com/azat)). +* Documentation low cardinality fix [#21086](https://github.com/ClickHouse/ClickHouse/pull/21086) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix __has_feature macros under gcc [#21088](https://github.com/ClickHouse/ClickHouse/pull/21088) ([Azat Khuzhin](https://github.com/azat)). +* test for window functions [#21094](https://github.com/ClickHouse/ClickHouse/pull/21094) ([Denny Crane](https://github.com/den-crane)). +* Fix replace[All] functions so that they don't generate garbage to stderr [#21098](https://github.com/ClickHouse/ClickHouse/pull/21098) ([Amos Bird](https://github.com/amosbird)). +* Better tests for protobuf format #2. [#21148](https://github.com/ClickHouse/ClickHouse/pull/21148) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ccache 4.2+ usage (RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK was not set) [#21158](https://github.com/ClickHouse/ClickHouse/pull/21158) ([Azat Khuzhin](https://github.com/azat)). +* Bump zookeeper version to 3.6.2 in tests [#21171](https://github.com/ClickHouse/ClickHouse/pull/21171) ([Azat Khuzhin](https://github.com/azat)). +* StorageRabbitMQ added UVLoop [#21193](https://github.com/ClickHouse/ClickHouse/pull/21193) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to make nukeeper better in single server mode [#21207](https://github.com/ClickHouse/ClickHouse/pull/21207) ([alesapin](https://github.com/alesapin)). +* A followup correction to [#19998](https://github.com/ClickHouse/ClickHouse/issues/19998) [#21221](https://github.com/ClickHouse/ClickHouse/pull/21221) ([Alexander Kazakov](https://github.com/Akazz)). +* Add tests for zstd and zlib http compression [#21279](https://github.com/ClickHouse/ClickHouse/pull/21279) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CheckConstraintsBlockOutputStream optimize nullable column case [#21285](https://github.com/ClickHouse/ClickHouse/pull/21285) ([Maksim Kita](https://github.com/kitaisreal)). +* Rewrite extractTextFromHTML function [#21292](https://github.com/ClickHouse/ClickHouse/pull/21292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abnormal server termination for nested writers [#21305](https://github.com/ClickHouse/ClickHouse/pull/21305) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unused writers [#21306](https://github.com/ClickHouse/ClickHouse/pull/21306) ([Azat Khuzhin](https://github.com/azat)). +* remove unused code in MergeTreeWriteAheadLog::restore [#21308](https://github.com/ClickHouse/ClickHouse/pull/21308) ([Fuwang Hu](https://github.com/fuwhu)). +* IColunm::hasEqualValues() [#21327](https://github.com/ClickHouse/ClickHouse/pull/21327) ([Amos Bird](https://github.com/amosbird)). +* AggregateFunctionSumMap better comment message [#21353](https://github.com/ClickHouse/ClickHouse/pull/21353) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse stop: wait for the server to be killed (process exited) [#21365](https://github.com/ClickHouse/ClickHouse/pull/21365) ([Azat Khuzhin](https://github.com/azat)). +* [ClickHouse][LOG]correct shutdown timeout log [#21366](https://github.com/ClickHouse/ClickHouse/pull/21366) ([jasong](https://github.com/songenjie)). +* fix a rare false negative in perf tests [#21381](https://github.com/ClickHouse/ClickHouse/pull/21381) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for issue[#21369](https://github.com/ClickHouse/ClickHouse/issues/21369) [#21387](https://github.com/ClickHouse/ClickHouse/pull/21387) ([Denny Crane](https://github.com/den-crane)). +* Add a test for [#14740](https://github.com/ClickHouse/ClickHouse/issues/14740) [#21392](https://github.com/ClickHouse/ClickHouse/pull/21392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15469](https://github.com/ClickHouse/ClickHouse/issues/15469) [#21393](https://github.com/ClickHouse/ClickHouse/pull/21393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in setting name [#21418](https://github.com/ClickHouse/ClickHouse/pull/21418) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix flaky tests with distributed queries [#21432](https://github.com/ClickHouse/ClickHouse/pull/21432) ([Azat Khuzhin](https://github.com/azat)). +* Fix ParsingException::displayText() [#21433](https://github.com/ClickHouse/ClickHouse/pull/21433) ([Azat Khuzhin](https://github.com/azat)). +* Use path as default prefix for coordination logs [#21439](https://github.com/ClickHouse/ClickHouse/pull/21439) ([alesapin](https://github.com/alesapin)). +* Try fix perftests. [#21447](https://github.com/ClickHouse/ClickHouse/pull/21447) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* avoid race in librdkafka [#21452](https://github.com/ClickHouse/ClickHouse/pull/21452) ([filimonov](https://github.com/filimonov)). +* test for [#21413](https://github.com/ClickHouse/ClickHouse/issues/21413) [#21455](https://github.com/ClickHouse/ClickHouse/pull/21455) ([Denny Crane](https://github.com/den-crane)). +* Tiny fix [#21458](https://github.com/ClickHouse/ClickHouse/pull/21458) ([Amos Bird](https://github.com/amosbird)). +* Just another fix for DDLWorker [#21461](https://github.com/ClickHouse/ClickHouse/pull/21461) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.3.10.1-lts.md b/docs/changelogs/v21.3.10.1-lts.md index 49ece009ad1..d43237428e2 100644 --- a/docs/changelogs/v21.3.10.1-lts.md +++ b/docs/changelogs/v21.3.10.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.10.1-lts FIXME as compared to v21.3.9.83-lts #### Bug Fix @@ -6,3 +13,7 @@ * Backported in [#23817](https://github.com/ClickHouse/ClickHouse/issues/23817): Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). * Backported in [#23814](https://github.com/ClickHouse/ClickHouse/issues/23814): Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.11.5-lts.md b/docs/changelogs/v21.3.11.5-lts.md index 61aa8a54688..2918ab860a6 100644 --- a/docs/changelogs/v21.3.11.5-lts.md +++ b/docs/changelogs/v21.3.11.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.11.5-lts FIXME as compared to v21.3.10.1-lts #### Improvement diff --git a/docs/changelogs/v21.3.12.2-lts.md b/docs/changelogs/v21.3.12.2-lts.md index cfddf8c9cdd..02c649284cc 100644 --- a/docs/changelogs/v21.3.12.2-lts.md +++ b/docs/changelogs/v21.3.12.2-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.12.2-lts FIXME as compared to v21.3.11.5-lts #### Bug Fix @@ -10,3 +17,9 @@ * Backported in [#24241](https://github.com/ClickHouse/ClickHouse/issues/24241): Fix wrong typo at StorageMemory, this bug was introduced at [#15127](https://github.com/ClickHouse/ClickHouse/issues/15127), now fixed, Closes [#24192](https://github.com/ClickHouse/ClickHouse/issues/24192). [#24193](https://github.com/ClickHouse/ClickHouse/pull/24193) ([张中南](https://github.com/plugine)). * Backported in [#24353](https://github.com/ClickHouse/ClickHouse/issues/24353): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.13.9-lts.md b/docs/changelogs/v21.3.13.9-lts.md index dccb9c53162..f78725e4408 100644 --- a/docs/changelogs/v21.3.13.9-lts.md +++ b/docs/changelogs/v21.3.13.9-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.13.9-lts FIXME as compared to v21.3.12.2-lts #### Improvement @@ -41,3 +48,8 @@ * NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.3: Remove endless `wait` from ZooKeeper client"'. [#24799](https://github.com/ClickHouse/ClickHouse/pull/24799) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.14.1-lts.md b/docs/changelogs/v21.3.14.1-lts.md index b2408471ccd..eb2ce73b01f 100644 --- a/docs/changelogs/v21.3.14.1-lts.md +++ b/docs/changelogs/v21.3.14.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.14.1-lts FIXME as compared to v21.3.13.9-lts #### Bug Fix @@ -8,3 +15,7 @@ * Backported in [#25716](https://github.com/ClickHouse/ClickHouse/issues/25716): `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#25712](https://github.com/ClickHouse/ClickHouse/issues/25712): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backport unrelated changes, which fixes aliases bug [#25681](https://github.com/ClickHouse/ClickHouse/pull/25681) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.3.15.4-stable.md b/docs/changelogs/v21.3.15.4-stable.md index c6b7a15aa4d..ca5a25850a1 100644 --- a/docs/changelogs/v21.3.15.4-stable.md +++ b/docs/changelogs/v21.3.15.4-stable.md @@ -1,6 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.15.4-stable FIXME as compared to v21.3.14.1-lts #### Bug Fix * Backported in [#25956](https://github.com/ClickHouse/ClickHouse/issues/25956): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). * Backported in [#26141](https://github.com/ClickHouse/ClickHouse/issues/26141): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.3 Docker unbundled fix [#26153](https://github.com/ClickHouse/ClickHouse/pull/26153) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.16.5-lts.md b/docs/changelogs/v21.3.16.5-lts.md index b5b31b64488..a1e73b7c019 100644 --- a/docs/changelogs/v21.3.16.5-lts.md +++ b/docs/changelogs/v21.3.16.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.16.5-lts FIXME as compared to v21.3.15.4-stable #### Bug Fix @@ -23,3 +30,15 @@ * Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). * Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.17.2-lts.md b/docs/changelogs/v21.3.17.2-lts.md index b6b48ddab61..41007a45634 100644 --- a/docs/changelogs/v21.3.17.2-lts.md +++ b/docs/changelogs/v21.3.17.2-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.17.2-lts FIXME as compared to v21.3.16.5-lts #### Bug Fix diff --git a/docs/changelogs/v21.3.18.4-lts.md b/docs/changelogs/v21.3.18.4-lts.md index 612b3660e3b..5bb29b08df8 100644 --- a/docs/changelogs/v21.3.18.4-lts.md +++ b/docs/changelogs/v21.3.18.4-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.18.4-lts FIXME as compared to v21.3.17.2-lts #### Improvement @@ -20,3 +27,10 @@ * Backported in [#30332](https://github.com/ClickHouse/ClickHouse/issues/30332): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). * Backported in [#30379](https://github.com/ClickHouse/ClickHouse/issues/30379): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.19.1-lts.md b/docs/changelogs/v21.3.19.1-lts.md index 6a2be8d6dcb..fa8e94fa7a1 100644 --- a/docs/changelogs/v21.3.19.1-lts.md +++ b/docs/changelogs/v21.3.19.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.19.1-lts FIXME as compared to v21.3.18.4-lts #### Performance Improvement @@ -24,3 +31,8 @@ * Backported in [#31894](https://github.com/ClickHouse/ClickHouse/issues/31894): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#32214](https://github.com/ClickHouse/ClickHouse/issues/32214): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.2.5-lts.md b/docs/changelogs/v21.3.2.5-lts.md index 5135f909b7f..0eb32439c6b 100644 --- a/docs/changelogs/v21.3.2.5-lts.md +++ b/docs/changelogs/v21.3.2.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.2.5-lts FIXME as compared to v21.2.1.5869-prestable #### Backward Incompatible Change @@ -159,3 +166,164 @@ * NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Pytest check [#18972](https://github.com/ClickHouse/ClickHouse/pull/18972) ([Ivan](https://github.com/abyss7)). +* [wip] support RANGE frame for window functions [#19299](https://github.com/ClickHouse/ClickHouse/pull/19299) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* In memory coordination inside ClickHouse [#19580](https://github.com/ClickHouse/ClickHouse/pull/19580) ([alesapin](https://github.com/alesapin)). +* client: more suggestions [#19584](https://github.com/ClickHouse/ClickHouse/pull/19584) ([Azat Khuzhin](https://github.com/azat)). +* Allow to run all style checks in one file [#19726](https://github.com/ClickHouse/ClickHouse/pull/19726) ([Anton Popov](https://github.com/CurtizJ)). +* Fix [#19371](https://github.com/ClickHouse/ClickHouse/issues/19371) [#19765](https://github.com/ClickHouse/ClickHouse/pull/19765) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor code improvements around ThreadStatus [#19771](https://github.com/ClickHouse/ClickHouse/pull/19771) ([Alexander Tokmakov](https://github.com/tavplubix)). +* continue of [#19487](https://github.com/ClickHouse/ClickHouse/issues/19487) [#19801](https://github.com/ClickHouse/ClickHouse/pull/19801) ([flynn](https://github.com/ucasfl)). +* Fix UBSan report in intDiv [#19876](https://github.com/ClickHouse/ClickHouse/pull/19876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow writing partial buffer [#19886](https://github.com/ClickHouse/ClickHouse/pull/19886) ([Azat Khuzhin](https://github.com/azat)). +* Remove an always-false condition from query parser [#19919](https://github.com/ClickHouse/ClickHouse/pull/19919) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UNBOUNDED FOLLOWING frame end [#19921](https://github.com/ClickHouse/ClickHouse/pull/19921) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix logical error in INSERT VALUES [#19925](https://github.com/ClickHouse/ClickHouse/pull/19925) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix test keeper integration tests [#19942](https://github.com/ClickHouse/ClickHouse/pull/19942) ([alesapin](https://github.com/alesapin)). +* Fix build [#19948](https://github.com/ClickHouse/ClickHouse/pull/19948) ([Kruglov Pavel](https://github.com/Avogar)). +* CURRENT ROW and offset for start of ROWS frame [#19951](https://github.com/ClickHouse/ClickHouse/pull/19951) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in geoHashesInBox [#19956](https://github.com/ClickHouse/ClickHouse/pull/19956) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MSan annotation for system.stack_trace [#19957](https://github.com/ClickHouse/ClickHouse/pull/19957) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration tests [#19971](https://github.com/ClickHouse/ClickHouse/pull/19971) ([Azat Khuzhin](https://github.com/azat)). +* Do not use inputs which values are known constants in ActionsDAG. [#19991](https://github.com/ClickHouse/ClickHouse/pull/19991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* update perf tests [#20000](https://github.com/ClickHouse/ClickHouse/pull/20000) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UBsan fixes [#20011](https://github.com/ClickHouse/ClickHouse/pull/20011) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#20029](https://github.com/ClickHouse/ClickHouse/pull/20029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update changelog [#20033](https://github.com/ClickHouse/ClickHouse/pull/20033) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Close input ports in DelayedPortsProcessor as soon as all outputs are finished [#20035](https://github.com/ClickHouse/ClickHouse/pull/20035) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ReadBuffer: check for unread data on next() [#20037](https://github.com/ClickHouse/ClickHouse/pull/20037) ([Ivan](https://github.com/abyss7)). +* ReadBuffer: check that buffer position is not set beyond end [#20039](https://github.com/ClickHouse/ClickHouse/pull/20039) ([Ivan](https://github.com/abyss7)). +* CURRENT ROW frame start for RANGE frame [#20041](https://github.com/ClickHouse/ClickHouse/pull/20041) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix log message with elapsed time while pushing to views [#20047](https://github.com/ClickHouse/ClickHouse/pull/20047) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in pointInPolygon [#20049](https://github.com/ClickHouse/ClickHouse/pull/20049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests *.reference files [#20050](https://github.com/ClickHouse/ClickHouse/pull/20050) ([Azat Khuzhin](https://github.com/azat)). +* ROWS OFFSET frame end [#20060](https://github.com/ClickHouse/ClickHouse/pull/20060) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add more tests for modulo of division of negative number [#20063](https://github.com/ClickHouse/ClickHouse/pull/20063) ([hexiaoting](https://github.com/hexiaoting)). +* detect unmarked long tests in flaky check [#20068](https://github.com/ClickHouse/ClickHouse/pull/20068) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix removing of filter column when split filter actions. [#20073](https://github.com/ClickHouse/ClickHouse/pull/20073) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove AddingConstColumn step and transform. [#20077](https://github.com/ClickHouse/ClickHouse/pull/20077) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* LimitReadBuffer: check that position always advances [#20078](https://github.com/ClickHouse/ClickHouse/pull/20078) ([Ivan](https://github.com/abyss7)). +* Add fuzzer for ColumnsDescription [#20094](https://github.com/ClickHouse/ClickHouse/pull/20094) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build actions dag to evaluate missing defaults. [#20097](https://github.com/ClickHouse/ClickHouse/pull/20097) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove never existing insert_in_memory_parts_timeout [#20099](https://github.com/ClickHouse/ClickHouse/pull/20099) ([Azat Khuzhin](https://github.com/azat)). +* RANGE OFFSET window frame [#20111](https://github.com/ClickHouse/ClickHouse/pull/20111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to drop readonly tables [#20114](https://github.com/ClickHouse/ClickHouse/pull/20114) ([nvartolomei](https://github.com/nvartolomei)). +* Add logging if Poco cannot allocate thread in tcp server [#20120](https://github.com/ClickHouse/ClickHouse/pull/20120) ([alesapin](https://github.com/alesapin)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Useless changes [#20150](https://github.com/ClickHouse/ClickHouse/pull/20150) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove some useless code [#20152](https://github.com/ClickHouse/ClickHouse/pull/20152) ([flynn](https://github.com/ucasfl)). +* Allow using MergeTreeWhereOptimizer not only with MergeTree-based storages [#20153](https://github.com/ClickHouse/ClickHouse/pull/20153) ([Max Akhmedov](https://github.com/zlobober)). +* Fix build of utils [#20155](https://github.com/ClickHouse/ClickHouse/pull/20155) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in arrayCumSum [#20160](https://github.com/ClickHouse/ClickHouse/pull/20160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.2 [#20185](https://github.com/ClickHouse/ClickHouse/pull/20185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ubsan error [#20204](https://github.com/ClickHouse/ClickHouse/pull/20204) ([Amos Bird](https://github.com/amosbird)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving [#20206](https://github.com/ClickHouse/ClickHouse/pull/20206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove dead code [#20212](https://github.com/ClickHouse/ClickHouse/pull/20212) ([nvartolomei](https://github.com/nvartolomei)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). +* Fix benign race in system.parts [#20226](https://github.com/ClickHouse/ClickHouse/pull/20226) ([alesapin](https://github.com/alesapin)). +* Add final to some classes [#20228](https://github.com/ClickHouse/ClickHouse/pull/20228) ([alesapin](https://github.com/alesapin)). +* refine code in MergeTreeData::loadDataParts to avoid parsing WAL file as data part [#20231](https://github.com/ClickHouse/ClickHouse/pull/20231) ([Fuwang Hu](https://github.com/fuwhu)). +* fix a problem in ArithmeticOperationsInAgrFuncOptimize [#20246](https://github.com/ClickHouse/ClickHouse/pull/20246) ([flynn](https://github.com/ucasfl)). +* Add retries to test_access_control_on_cluster [#20247](https://github.com/ClickHouse/ClickHouse/pull/20247) ([Ilya Yatsishin](https://github.com/qoega)). +* Stable sort for test cases in test_dictionaries_all_layouts_separate_sources [#20248](https://github.com/ClickHouse/ClickHouse/pull/20248) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix not closed ports in DelayedPortsProcessor [#20251](https://github.com/ClickHouse/ClickHouse/pull/20251) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* print changed settings in fuzzer when the server dies [#20281](https://github.com/ClickHouse/ClickHouse/pull/20281) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Put windows in such an order that we can sort less [#20284](https://github.com/ClickHouse/ClickHouse/pull/20284) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Aggregate function deltaSum use restrict keyword [#20287](https://github.com/ClickHouse/ClickHouse/pull/20287) ([Maksim Kita](https://github.com/kitaisreal)). +* make window functions faster [#20293](https://github.com/ClickHouse/ClickHouse/pull/20293) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Accept arbitrary numeric types for numbers() arguments (for scientific notation) [#20301](https://github.com/ClickHouse/ClickHouse/pull/20301) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00738_lock_for_inner_table flakiness [#20305](https://github.com/ClickHouse/ClickHouse/pull/20305) ([Azat Khuzhin](https://github.com/azat)). +* Filter push down [#20341](https://github.com/ClickHouse/ClickHouse/pull/20341) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Common types template instantiations [#20348](https://github.com/ClickHouse/ClickHouse/pull/20348) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for sign column drop [#20398](https://github.com/ClickHouse/ClickHouse/pull/20398) ([alesapin](https://github.com/alesapin)). +* Merging [#19204](https://github.com/ClickHouse/ClickHouse/issues/19204) [#20399](https://github.com/ClickHouse/ClickHouse/pull/20399) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix 01700_system_zookeeper_path_in [#20449](https://github.com/ClickHouse/ClickHouse/pull/20449) ([alesapin](https://github.com/alesapin)). +* Print stack trace on SIGTRAP [#20453](https://github.com/ClickHouse/ClickHouse/pull/20453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in arrayDifference [#20458](https://github.com/ClickHouse/ClickHouse/pull/20458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress UBSan report in Decimal comparison [#20459](https://github.com/ClickHouse/ClickHouse/pull/20459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve backtrace catching on server failures in CI for stress tests [#20462](https://github.com/ClickHouse/ClickHouse/pull/20462) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server terminations due to write failures [#20465](https://github.com/ClickHouse/ClickHouse/pull/20465) ([Azat Khuzhin](https://github.com/azat)). +* Improve logging during reading from MergeTree [#20466](https://github.com/ClickHouse/ClickHouse/pull/20466) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in intDiv [#20475](https://github.com/ClickHouse/ClickHouse/pull/20475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* (unrealistic case, never possible in prod) HashTable fix potential bug during resize with nonstandard grower [#20489](https://github.com/ClickHouse/ClickHouse/pull/20489) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8654](https://github.com/ClickHouse/ClickHouse/issues/8654) [#20492](https://github.com/ClickHouse/ClickHouse/pull/20492) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#10893](https://github.com/ClickHouse/ClickHouse/issues/10893) [#20493](https://github.com/ClickHouse/ClickHouse/pull/20493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable testkeeper snapshots for tests [#20496](https://github.com/ClickHouse/ClickHouse/pull/20496) ([alesapin](https://github.com/alesapin)). +* Fix non-zero session reconnect in integration test [#20501](https://github.com/ClickHouse/ClickHouse/pull/20501) ([alesapin](https://github.com/alesapin)). +* Reduce test scale [#20511](https://github.com/ClickHouse/ClickHouse/pull/20511) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Quick build fix [#20513](https://github.com/ClickHouse/ClickHouse/pull/20513) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace null fields in tuple during parsing with default values [#20541](https://github.com/ClickHouse/ClickHouse/pull/20541) ([Maksim Kita](https://github.com/kitaisreal)). +* rabbitmq: add missing format factory settings [#20545](https://github.com/ClickHouse/ClickHouse/pull/20545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Persistent coordination log storage [#20585](https://github.com/ClickHouse/ClickHouse/pull/20585) ([alesapin](https://github.com/alesapin)). +* Add hung check to stress test [#20588](https://github.com/ClickHouse/ClickHouse/pull/20588) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ignore data store files [#20598](https://github.com/ClickHouse/ClickHouse/pull/20598) ([tison](https://github.com/tisonkun)). +* Dictionary create source with functions crash fix [#20623](https://github.com/ClickHouse/ClickHouse/pull/20623) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix fasttest retry for failed tests [#20627](https://github.com/ClickHouse/ClickHouse/pull/20627) ([alesapin](https://github.com/alesapin)). +* Don't backport base commit of branch in the same branch [#20628](https://github.com/ClickHouse/ClickHouse/pull/20628) ([Ivan](https://github.com/abyss7)). +* Add test for already fixed odbc Postgres date type conversion [#20712](https://github.com/ClickHouse/ClickHouse/pull/20712) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve backtrace catching on server failures in CI for fast-tests [#20864](https://github.com/ClickHouse/ClickHouse/pull/20864) ([Azat Khuzhin](https://github.com/azat)). +* Better postgres db engine numeric conversion [#20874](https://github.com/ClickHouse/ClickHouse/pull/20874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix undefined-behavior in ReservoirSamplerDeterministic.h [#20879](https://github.com/ClickHouse/ClickHouse/pull/20879) ([Vitaly Baranov](https://github.com/vitlibar)). +* ccache 4.2+ does not requires any quirks for SOURCE_DATE_EPOCH [#20883](https://github.com/ClickHouse/ClickHouse/pull/20883) ([Azat Khuzhin](https://github.com/azat)). +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix push to dockerhub [#20960](https://github.com/ClickHouse/ClickHouse/pull/20960) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better tests for protobuf format. [#20968](https://github.com/ClickHouse/ClickHouse/pull/20968) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function sumMap decimal fix [#20970](https://github.com/ClickHouse/ClickHouse/pull/20970) ([Maksim Kita](https://github.com/kitaisreal)). +* test for decimal ( p , s) in dictionaries [#20980](https://github.com/ClickHouse/ClickHouse/pull/20980) ([Denny Crane](https://github.com/den-crane)). +* Fix uncaught exception when HTTP client goes away [#20981](https://github.com/ClickHouse/ClickHouse/pull/20981) ([Azat Khuzhin](https://github.com/azat)). +* Increase buffer for uncaught exception / std::terminate [#20989](https://github.com/ClickHouse/ClickHouse/pull/20989) ([Azat Khuzhin](https://github.com/azat)). +* Constraints complex types support [#20990](https://github.com/ClickHouse/ClickHouse/pull/20990) ([Maksim Kita](https://github.com/kitaisreal)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving 2 [#20995](https://github.com/ClickHouse/ClickHouse/pull/20995) ([Amos Bird](https://github.com/amosbird)). +* Add log message when stacktrace cannot be obtained for thread [#20996](https://github.com/ClickHouse/ClickHouse/pull/20996) ([Azat Khuzhin](https://github.com/azat)). +* Preserve mysql logs in test_materialize_mysql_database [#21016](https://github.com/ClickHouse/ClickHouse/pull/21016) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes in Decimal [#21017](https://github.com/ClickHouse/ClickHouse/pull/21017) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* yamllint style check [#21019](https://github.com/ClickHouse/ClickHouse/pull/21019) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer size for trace collection [#21020](https://github.com/ClickHouse/ClickHouse/pull/21020) ([Azat Khuzhin](https://github.com/azat)). +* Try fix MaterializeMySQL integration test [#21021](https://github.com/ClickHouse/ClickHouse/pull/21021) ([Winter Zhang](https://github.com/zhang2014)). +* Fix performance tests (by avoid sharing status file for right and left server) [#21022](https://github.com/ClickHouse/ClickHouse/pull/21022) ([Azat Khuzhin](https://github.com/azat)). +* Revert "optimize aggfunc column data copy ([#19407](https://github.com/ClickHouse/ClickHouse/issues/19407))" [#21024](https://github.com/ClickHouse/ClickHouse/pull/21024) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Decimal arithmetic [#21025](https://github.com/ClickHouse/ClickHouse/pull/21025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update lowcardinality.md [#21033](https://github.com/ClickHouse/ClickHouse/pull/21033) ([Michael Monashev](https://github.com/MichaelMonashev)). +* Fix DateTime64 from Float [#21050](https://github.com/ClickHouse/ClickHouse/pull/21050) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#19376](https://github.com/ClickHouse/ClickHouse/issues/19376) [#21051](https://github.com/ClickHouse/ClickHouse/pull/21051) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#20620](https://github.com/ClickHouse/ClickHouse/issues/20620) [#21052](https://github.com/ClickHouse/ClickHouse/pull/21052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9781](https://github.com/ClickHouse/ClickHouse/issues/9781) [#21074](https://github.com/ClickHouse/ClickHouse/pull/21074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix various issues in DDLWorker (SIGSEGV and others) [#21079](https://github.com/ClickHouse/ClickHouse/pull/21079) ([Azat Khuzhin](https://github.com/azat)). +* Documentation low cardinality fix [#21086](https://github.com/ClickHouse/ClickHouse/pull/21086) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix __has_feature macros under gcc [#21088](https://github.com/ClickHouse/ClickHouse/pull/21088) ([Azat Khuzhin](https://github.com/azat)). +* test for window functions [#21094](https://github.com/ClickHouse/ClickHouse/pull/21094) ([Denny Crane](https://github.com/den-crane)). +* Fix replace[All] functions so that they don't generate garbage to stderr [#21098](https://github.com/ClickHouse/ClickHouse/pull/21098) ([Amos Bird](https://github.com/amosbird)). +* Better tests for protobuf format #2. [#21148](https://github.com/ClickHouse/ClickHouse/pull/21148) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ccache 4.2+ usage (RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK was not set) [#21158](https://github.com/ClickHouse/ClickHouse/pull/21158) ([Azat Khuzhin](https://github.com/azat)). +* Bump zookeeper version to 3.6.2 in tests [#21171](https://github.com/ClickHouse/ClickHouse/pull/21171) ([Azat Khuzhin](https://github.com/azat)). +* StorageRabbitMQ added UVLoop [#21193](https://github.com/ClickHouse/ClickHouse/pull/21193) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to make nukeeper better in single server mode [#21207](https://github.com/ClickHouse/ClickHouse/pull/21207) ([alesapin](https://github.com/alesapin)). +* A followup correction to [#19998](https://github.com/ClickHouse/ClickHouse/issues/19998) [#21221](https://github.com/ClickHouse/ClickHouse/pull/21221) ([Alexander Kazakov](https://github.com/Akazz)). +* Add tests for zstd and zlib http compression [#21279](https://github.com/ClickHouse/ClickHouse/pull/21279) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CheckConstraintsBlockOutputStream optimize nullable column case [#21285](https://github.com/ClickHouse/ClickHouse/pull/21285) ([Maksim Kita](https://github.com/kitaisreal)). +* Rewrite extractTextFromHTML function [#21292](https://github.com/ClickHouse/ClickHouse/pull/21292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abnormal server termination for nested writers [#21305](https://github.com/ClickHouse/ClickHouse/pull/21305) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unused writers [#21306](https://github.com/ClickHouse/ClickHouse/pull/21306) ([Azat Khuzhin](https://github.com/azat)). +* remove unused code in MergeTreeWriteAheadLog::restore [#21308](https://github.com/ClickHouse/ClickHouse/pull/21308) ([Fuwang Hu](https://github.com/fuwhu)). +* IColunm::hasEqualValues() [#21327](https://github.com/ClickHouse/ClickHouse/pull/21327) ([Amos Bird](https://github.com/amosbird)). +* AggregateFunctionSumMap better comment message [#21353](https://github.com/ClickHouse/ClickHouse/pull/21353) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse stop: wait for the server to be killed (process exited) [#21365](https://github.com/ClickHouse/ClickHouse/pull/21365) ([Azat Khuzhin](https://github.com/azat)). +* [ClickHouse][LOG]correct shutdown timeout log [#21366](https://github.com/ClickHouse/ClickHouse/pull/21366) ([jasong](https://github.com/songenjie)). +* fix a rare false negative in perf tests [#21381](https://github.com/ClickHouse/ClickHouse/pull/21381) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for issue[#21369](https://github.com/ClickHouse/ClickHouse/issues/21369) [#21387](https://github.com/ClickHouse/ClickHouse/pull/21387) ([Denny Crane](https://github.com/den-crane)). +* Add a test for [#14740](https://github.com/ClickHouse/ClickHouse/issues/14740) [#21392](https://github.com/ClickHouse/ClickHouse/pull/21392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15469](https://github.com/ClickHouse/ClickHouse/issues/15469) [#21393](https://github.com/ClickHouse/ClickHouse/pull/21393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in setting name [#21418](https://github.com/ClickHouse/ClickHouse/pull/21418) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix flaky tests with distributed queries [#21432](https://github.com/ClickHouse/ClickHouse/pull/21432) ([Azat Khuzhin](https://github.com/azat)). +* Fix ParsingException::displayText() [#21433](https://github.com/ClickHouse/ClickHouse/pull/21433) ([Azat Khuzhin](https://github.com/azat)). +* Use path as default prefix for coordination logs [#21439](https://github.com/ClickHouse/ClickHouse/pull/21439) ([alesapin](https://github.com/alesapin)). +* Try fix perftests. [#21447](https://github.com/ClickHouse/ClickHouse/pull/21447) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* avoid race in librdkafka [#21452](https://github.com/ClickHouse/ClickHouse/pull/21452) ([filimonov](https://github.com/filimonov)). +* test for [#21413](https://github.com/ClickHouse/ClickHouse/issues/21413) [#21455](https://github.com/ClickHouse/ClickHouse/pull/21455) ([Denny Crane](https://github.com/den-crane)). +* Tiny fix [#21458](https://github.com/ClickHouse/ClickHouse/pull/21458) ([Amos Bird](https://github.com/amosbird)). +* Just another fix for DDLWorker [#21461](https://github.com/ClickHouse/ClickHouse/pull/21461) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable hedged requests for release 21.3 [#21534](https://github.com/ClickHouse/ClickHouse/pull/21534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.20.1-lts.md b/docs/changelogs/v21.3.20.1-lts.md index ac8c7d2ece2..19cca4babf4 100644 --- a/docs/changelogs/v21.3.20.1-lts.md +++ b/docs/changelogs/v21.3.20.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.20.1-lts FIXME as compared to v21.3.19.1-lts #### Bug Fix @@ -8,3 +15,13 @@ * Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cherry pick [#33065](https://github.com/ClickHouse/ClickHouse/issues/33065) to 21.3: Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33079](https://github.com/ClickHouse/ClickHouse/issues/33079) [#33578](https://github.com/ClickHouse/ClickHouse/pull/33578) ([Heena Bansal](https://github.com/HeenaBansal2009)). + diff --git a/docs/changelogs/v21.3.3.14-lts.md b/docs/changelogs/v21.3.3.14-lts.md index 3dbfd7f0a04..9b99c6c1b5c 100644 --- a/docs/changelogs/v21.3.3.14-lts.md +++ b/docs/changelogs/v21.3.3.14-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.3.14-lts FIXME as compared to v21.3.2.5-lts #### Bug Fix @@ -9,3 +16,7 @@ * Backported in [#21883](https://github.com/ClickHouse/ClickHouse/issues/21883): Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). * Backported in [#21874](https://github.com/ClickHouse/ClickHouse/issues/21874): Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.4.25-lts.md b/docs/changelogs/v21.3.4.25-lts.md index 431c498dbed..dee9e6a48b3 100644 --- a/docs/changelogs/v21.3.4.25-lts.md +++ b/docs/changelogs/v21.3.4.25-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.4.25-lts FIXME as compared to v21.3.3.14-lts #### Bug Fix @@ -16,3 +23,8 @@ * Backported in [#22185](https://github.com/ClickHouse/ClickHouse/issues/22185): Fix uncaught exception in InterserverIOHTTPHandler. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#22205](https://github.com/ClickHouse/ClickHouse/issues/22205): Use finalize() over next() for nested writers. [#22147](https://github.com/ClickHouse/ClickHouse/pull/22147) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* 21.3: quick fix for broken resolution of apt.llvm.org on Yandex infra [#22047](https://github.com/ClickHouse/ClickHouse/pull/22047) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.5.42-lts.md b/docs/changelogs/v21.3.5.42-lts.md index d8616efd0ff..bc68841062c 100644 --- a/docs/changelogs/v21.3.5.42-lts.md +++ b/docs/changelogs/v21.3.5.42-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.5.42-lts FIXME as compared to v21.3.4.25-lts #### Bug Fix @@ -25,3 +32,7 @@ * Backported in [#22653](https://github.com/ClickHouse/ClickHouse/issues/22653): Try flush write buffer only if it is initialized. Fixes segfault when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#22681](https://github.com/ClickHouse/ClickHouse/issues/22681): Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Manual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.3 [#22504](https://github.com/ClickHouse/ClickHouse/pull/22504) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.3.6.55-lts.md b/docs/changelogs/v21.3.6.55-lts.md index 68980af6264..3a517f7386f 100644 --- a/docs/changelogs/v21.3.6.55-lts.md +++ b/docs/changelogs/v21.3.6.55-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.6.55-lts FIXME as compared to v21.3.5.42-lts #### Improvement @@ -17,3 +24,7 @@ * Backported in [#22916](https://github.com/ClickHouse/ClickHouse/issues/22916): LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#22920](https://github.com/ClickHouse/ClickHouse/issues/22920): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.7.62-stable.md b/docs/changelogs/v21.3.7.62-stable.md index df919be0f42..24f759ca3c0 100644 --- a/docs/changelogs/v21.3.7.62-stable.md +++ b/docs/changelogs/v21.3.7.62-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.7.62-stable FIXME as compared to v21.3.6.55-lts #### Improvement @@ -10,3 +17,7 @@ * Backported in [#23075](https://github.com/ClickHouse/ClickHouse/issues/23075): Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#23170](https://github.com/ClickHouse/ClickHouse/issues/23170): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.8.76-lts.md b/docs/changelogs/v21.3.8.76-lts.md index 2002d9e3e1f..acb0b99bae5 100644 --- a/docs/changelogs/v21.3.8.76-lts.md +++ b/docs/changelogs/v21.3.8.76-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.8.76-lts FIXME as compared to v21.3.7.62-stable #### Improvement @@ -19,3 +26,7 @@ * Backported in [#23536](https://github.com/ClickHouse/ClickHouse/issues/23536): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). * Backported in [#23533](https://github.com/ClickHouse/ClickHouse/issues/23533): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.3.9.83-lts.md b/docs/changelogs/v21.3.9.83-lts.md index e437ee4800f..450faeea3f2 100644 --- a/docs/changelogs/v21.3.9.83-lts.md +++ b/docs/changelogs/v21.3.9.83-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.9.83-lts FIXME as compared to v21.3.8.76-lts #### Improvement diff --git a/docs/changelogs/v21.4.1.6422-prestable.md b/docs/changelogs/v21.4.1.6422-prestable.md index 8cd10834fae..684fde2e3b7 100644 --- a/docs/changelogs/v21.4.1.6422-prestable.md +++ b/docs/changelogs/v21.4.1.6422-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.1.6422-prestable FIXME as compared to v21.3.1.6185-prestable #### Backward Incompatible Change @@ -167,6 +174,117 @@ * NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). * NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add soft task timeout for Intergation tests [#16608](https://github.com/ClickHouse/ClickHouse/pull/16608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging geometry functions [#19257](https://github.com/ClickHouse/ClickHouse/pull/19257) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove outdated suppressions, part 2 [#19496](https://github.com/ClickHouse/ClickHouse/pull/19496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move some CI-related scripts to github [#20946](https://github.com/ClickHouse/ClickHouse/pull/20946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170) [#21182](https://github.com/ClickHouse/ClickHouse/pull/21182) ([Tachikoma](https://github.com/ikarishinjieva)). +* Add more tests for quota consumption by the SHOW statement [#21190](https://github.com/ClickHouse/ClickHouse/pull/21190) ([Vitaly Baranov](https://github.com/vitlibar)). +* Save packed keys for GROUP BY with multiple fixed size keys [#21196](https://github.com/ClickHouse/ClickHouse/pull/21196) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refine MergeTreeData::loadDataParts to not parse format version file and detached directory [#21351](https://github.com/ClickHouse/ClickHouse/pull/21351) ([Fuwang Hu](https://github.com/fuwhu)). +* Persistent nukeeper snapshot storage [#21425](https://github.com/ClickHouse/ClickHouse/pull/21425) ([alesapin](https://github.com/alesapin)). +* Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) [#21436](https://github.com/ClickHouse/ClickHouse/pull/21436) ([Azat Khuzhin](https://github.com/azat)). +* Adjust prewhere_with_row_level_filter performance test [#21442](https://github.com/ClickHouse/ClickHouse/pull/21442) ([Denis Glazachev](https://github.com/traceon)). +* Refactor actions dag [#21459](https://github.com/ClickHouse/ClickHouse/pull/21459) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* add query formatting idempotence check to fuzzer [#21466](https://github.com/ClickHouse/ClickHouse/pull/21466) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix heap-buffer-overflow in highlighting multi-line comments [#21492](https://github.com/ClickHouse/ClickHouse/pull/21492) ([Azat Khuzhin](https://github.com/azat)). +* Fix global stop merges in test [#21508](https://github.com/ClickHouse/ClickHouse/pull/21508) ([alesapin](https://github.com/alesapin)). +* DirectDictionary updated [#21513](https://github.com/ClickHouse/ClickHouse/pull/21513) ([Maksim Kita](https://github.com/kitaisreal)). +* Avoid processing optimize_skip_unused_shards twice [#21526](https://github.com/ClickHouse/ClickHouse/pull/21526) ([Azat Khuzhin](https://github.com/azat)). +* DOCSUP-7197: Escaped Unicode replaced with symbols [#21530](https://github.com/ClickHouse/ClickHouse/pull/21530) ([lehasm](https://github.com/lehasm)). +* Pod array left pad not multiple of element crash fix [#21532](https://github.com/ClickHouse/ClickHouse/pull/21532) ([Maksim Kita](https://github.com/kitaisreal)). +* ShellCommand waitpid eintr signal fix [#21546](https://github.com/ClickHouse/ClickHouse/pull/21546) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactoring of data types serialization [#21562](https://github.com/ClickHouse/ClickHouse/pull/21562) ([Anton Popov](https://github.com/CurtizJ)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added specialized CacheDictionaryStorage [#21573](https://github.com/ClickHouse/ClickHouse/pull/21573) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Union merge for arcadia_skip_list.txt to avoid frequent conflicts [#21580](https://github.com/ClickHouse/ClickHouse/pull/21580) ([Azat Khuzhin](https://github.com/azat)). +* Enable ipv6 in NuRaft [#21593](https://github.com/ClickHouse/ClickHouse/pull/21593) ([alesapin](https://github.com/alesapin)). +* add an article about ast-based fuzzer [#21608](https://github.com/ClickHouse/ClickHouse/pull/21608) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.3 [#21618](https://github.com/ClickHouse/ClickHouse/pull/21618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* redefine some errorcode [#21629](https://github.com/ClickHouse/ClickHouse/pull/21629) ([flynn](https://github.com/ucasfl)). +* Fix ambigous column error in joins_in_memory [#21658](https://github.com/ClickHouse/ClickHouse/pull/21658) ([Vladimir C](https://github.com/vdimir)). +* Add test for path as a query parameter in system.zookeeper [#21661](https://github.com/ClickHouse/ClickHouse/pull/21661) ([Kruglov Pavel](https://github.com/Avogar)). +* ExecutablePool fix default max execution time setting [#21662](https://github.com/ClickHouse/ClickHouse/pull/21662) ([Maksim Kita](https://github.com/kitaisreal)). +* DictionaryStructure fix non unique attribute names [#21668](https://github.com/ClickHouse/ClickHouse/pull/21668) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky test_replace_partition [#21674](https://github.com/ClickHouse/ClickHouse/pull/21674) ([Azat Khuzhin](https://github.com/azat)). +* DOC Fix ORDER BY syntax [#21675](https://github.com/ClickHouse/ClickHouse/pull/21675) ([Michael Monashev](https://github.com/MichaelMonashev)). +* PODArray swap fix [#21678](https://github.com/ClickHouse/ClickHouse/pull/21678) ([Maksim Kita](https://github.com/kitaisreal)). +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). +* Run three nodes with Replicated database and NuKeeper in CI [#21690](https://github.com/ClickHouse/ClickHouse/pull/21690) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error message in clickhouse-test [#21691](https://github.com/ClickHouse/ClickHouse/pull/21691) ([Azat Khuzhin](https://github.com/azat)). +* Set SOCK_CLOEXEC for sockets (hardcoded via poco update) [#21695](https://github.com/ClickHouse/ClickHouse/pull/21695) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes (that was found by stress tests) [#21696](https://github.com/ClickHouse/ClickHouse/pull/21696) ([Azat Khuzhin](https://github.com/azat)). +* Fix log_comment for *.sh in clickhouse-test [#21700](https://github.com/ClickHouse/ClickHouse/pull/21700) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless CMake option [#21712](https://github.com/ClickHouse/ClickHouse/pull/21712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in modulo by constant [#21713](https://github.com/ClickHouse/ClickHouse/pull/21713) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more variants for memcpy benchmark [#21715](https://github.com/ClickHouse/ClickHouse/pull/21715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests [#21724](https://github.com/ClickHouse/ClickHouse/pull/21724) ([Azat Khuzhin](https://github.com/azat)). +* make the fuzzer use sources from the CI [#21754](https://github.com/ClickHouse/ClickHouse/pull/21754) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add one more variant to memcpy benchmark [#21759](https://github.com/ClickHouse/ClickHouse/pull/21759) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* docs(fix): typo [#21775](https://github.com/ClickHouse/ClickHouse/pull/21775) ([Ali Demirci](https://github.com/depyronick)). +* DDLWorker.cpp: fixed exceeded amount of tries typo [#21807](https://github.com/ClickHouse/ClickHouse/pull/21807) ([Eldar Nasyrov](https://github.com/3ldar-nasyrov)). +* fix integration MaterializeMySQL test [#21819](https://github.com/ClickHouse/ClickHouse/pull/21819) ([TCeason](https://github.com/TCeason)). +* more robust error handling in perf test [#21846](https://github.com/ClickHouse/ClickHouse/pull/21846) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#17302](https://github.com/ClickHouse/ClickHouse/issues/17302) [#21848](https://github.com/ClickHouse/ClickHouse/pull/21848) ([Denny Crane](https://github.com/den-crane)). +* Add bash completion support for clickhouse utils [#21853](https://github.com/ClickHouse/ClickHouse/pull/21853) ([Azat Khuzhin](https://github.com/azat)). +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* fix fuzzer failure in tupleElement formatting [#21896](https://github.com/ClickHouse/ClickHouse/pull/21896) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix possibly dangling reference to Context [#21913](https://github.com/ClickHouse/ClickHouse/pull/21913) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add stress test for distributed queries [#21944](https://github.com/ClickHouse/ClickHouse/pull/21944) ([Azat Khuzhin](https://github.com/azat)). +* Fix misleading log in WriteBufferFromS3 [#21954](https://github.com/ClickHouse/ClickHouse/pull/21954) ([flynn](https://github.com/ucasfl)). +* Add a test for [#21991](https://github.com/ClickHouse/ClickHouse/issues/21991) [#21995](https://github.com/ClickHouse/ClickHouse/pull/21995) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11720](https://github.com/ClickHouse/ClickHouse/issues/11720) [#21997](https://github.com/ClickHouse/ClickHouse/pull/21997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15784](https://github.com/ClickHouse/ClickHouse/issues/15784) [#22002](https://github.com/ClickHouse/ClickHouse/pull/22002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* prevent accidental reinterpret_cast in Field::get<> [#22003](https://github.com/ClickHouse/ClickHouse/pull/22003) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in addMonths [#22006](https://github.com/ClickHouse/ClickHouse/pull/22006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#7963](https://github.com/ClickHouse/ClickHouse/issues/7963) [#22007](https://github.com/ClickHouse/ClickHouse/pull/22007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in intDiv [#21769](https://github.com/ClickHouse/ClickHouse/issues/21769) [#22009](https://github.com/ClickHouse/ClickHouse/pull/22009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cast to enum nullable fix [#22026](https://github.com/ClickHouse/ClickHouse/pull/22026) ([Maksim Kita](https://github.com/kitaisreal)). +* Small simplification in ExternalLoader. [#22027](https://github.com/ClickHouse/ClickHouse/pull/22027) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for [#21760](https://github.com/ClickHouse/ClickHouse/issues/21760) [#22036](https://github.com/ClickHouse/ClickHouse/pull/22036) ([Anton Popov](https://github.com/CurtizJ)). +* quick fix for broken resolution of apt.llvm.org on Yandex infra [#22055](https://github.com/ClickHouse/ClickHouse/pull/22055) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Darwin cmake disable memcpy benchmark [#22056](https://github.com/ClickHouse/ClickHouse/pull/22056) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix UBSan report in TransformDateTime64 [#22062](https://github.com/ClickHouse/ClickHouse/pull/22062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in mapPopulateSeries. [#22099](https://github.com/ClickHouse/ClickHouse/pull/22099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bar with invalid float value [#22114](https://github.com/ClickHouse/ClickHouse/pull/22114) ([flynn](https://github.com/ucasfl)). +* remove useless code [#22117](https://github.com/ClickHouse/ClickHouse/pull/22117) ([flynn](https://github.com/ucasfl)). +* stable formatting for negate() [#22133](https://github.com/ClickHouse/ClickHouse/pull/22133) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* adjust perf test thresholds [#22148](https://github.com/ClickHouse/ClickHouse/pull/22148) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix sleep_in_send_tables_status_ms/sleep_in_send_data_ms in integration tests [#22151](https://github.com/ClickHouse/ClickHouse/pull/22151) ([Azat Khuzhin](https://github.com/azat)). +* Update requirements.txt [#22153](https://github.com/ClickHouse/ClickHouse/pull/22153) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some flaky order dependent integration tests. [#22170](https://github.com/ClickHouse/ClickHouse/pull/22170) ([alesapin](https://github.com/alesapin)). +* Prevent busy waiting in hedged requests when async_socket_for_remote=0 [#22172](https://github.com/ClickHouse/ClickHouse/pull/22172) ([Kruglov Pavel](https://github.com/Avogar)). +* less flaky functional tests [#22181](https://github.com/ClickHouse/ClickHouse/pull/22181) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489) [#22219](https://github.com/ClickHouse/ClickHouse/pull/22219) ([Denny Crane](https://github.com/den-crane)). +* CachedCompressedReadBuffer fix cache usage [#22225](https://github.com/ClickHouse/ClickHouse/pull/22225) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix MSan report in `quantileDeterministic` [#22235](https://github.com/ClickHouse/ClickHouse/pull/22235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove harmful default parameters [#22238](https://github.com/ClickHouse/ClickHouse/pull/22238) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build error [#22243](https://github.com/ClickHouse/ClickHouse/pull/22243) ([hexiaoting](https://github.com/hexiaoting)). +* Rename NuKeeper and TestKeeper to Keeper in all places [#22274](https://github.com/ClickHouse/ClickHouse/pull/22274) ([alesapin](https://github.com/alesapin)). +* Update materialize-mysql.md [#22275](https://github.com/ClickHouse/ClickHouse/pull/22275) ([曲正鹏](https://github.com/quzhengpeng)). +* Fix native macOS build for ALL_BUILD (Xcode/AppleClang) [#22289](https://github.com/ClickHouse/ClickHouse/pull/22289) ([Denis Glazachev](https://github.com/traceon)). +* Add suffixes for dockerfile arguments [#22301](https://github.com/ClickHouse/ClickHouse/pull/22301) ([filimonov](https://github.com/filimonov)). +* More coarse test for DateLUT [#22320](https://github.com/ClickHouse/ClickHouse/pull/22320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#22328](https://github.com/ClickHouse/ClickHouse/pull/22328) ([Anton Popov](https://github.com/CurtizJ)). +* Maybe fix false MSan report in GRPC [#22338](https://github.com/ClickHouse/ClickHouse/pull/22338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 3) [#22357](https://github.com/ClickHouse/ClickHouse/pull/22357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 4) [#22358](https://github.com/ClickHouse/ClickHouse/pull/22358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky tests test_row_policy* and test_quota* [#22371](https://github.com/ClickHouse/ClickHouse/pull/22371) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix flaky rabbitmq test [#22380](https://github.com/ClickHouse/ClickHouse/pull/22380) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix UBSan report in mapOp [#22389](https://github.com/ClickHouse/ClickHouse/pull/22389) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove recursive submodules from Arrow [#22390](https://github.com/ClickHouse/ClickHouse/pull/22390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some OOMs in stress tests [#22396](https://github.com/ClickHouse/ClickHouse/pull/22396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly place debug helpers [#22407](https://github.com/ClickHouse/ClickHouse/pull/22407) ([Amos Bird](https://github.com/amosbird)). +* fix error message for invalid window frame start [#22412](https://github.com/ClickHouse/ClickHouse/pull/22412) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flapping test_s3_zero_copy_replication [#22440](https://github.com/ClickHouse/ClickHouse/pull/22440) ([ianton-ru](https://github.com/ianton-ru)). +* Lower scale of a test [#22446](https://github.com/ClickHouse/ClickHouse/pull/22446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust Keeper settings for CI [#22470](https://github.com/ClickHouse/ClickHouse/pull/22470) ([Alexander Tokmakov](https://github.com/tavplubix)). +* try clang 11 in fast test [#22472](https://github.com/ClickHouse/ClickHouse/pull/22472) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove TestFlows (2) [#22480](https://github.com/ClickHouse/ClickHouse/pull/22480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature (datasketches support in clickhouse #14893) * Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.4.2.10-prestable.md b/docs/changelogs/v21.4.2.10-prestable.md index 1bc440d126c..11767613551 100644 --- a/docs/changelogs/v21.4.2.10-prestable.md +++ b/docs/changelogs/v21.4.2.10-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.2.10-prestable FIXME as compared to v21.3.1.6185-prestable #### Backward Incompatible Change @@ -170,6 +177,114 @@ * NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). * NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add soft task timeout for Intergation tests [#16608](https://github.com/ClickHouse/ClickHouse/pull/16608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging geometry functions [#19257](https://github.com/ClickHouse/ClickHouse/pull/19257) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove outdated suppressions, part 2 [#19496](https://github.com/ClickHouse/ClickHouse/pull/19496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move some CI-related scripts to github [#20946](https://github.com/ClickHouse/ClickHouse/pull/20946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170) [#21182](https://github.com/ClickHouse/ClickHouse/pull/21182) ([Tachikoma](https://github.com/ikarishinjieva)). +* Add more tests for quota consumption by the SHOW statement [#21190](https://github.com/ClickHouse/ClickHouse/pull/21190) ([Vitaly Baranov](https://github.com/vitlibar)). +* Save packed keys for GROUP BY with multiple fixed size keys [#21196](https://github.com/ClickHouse/ClickHouse/pull/21196) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refine MergeTreeData::loadDataParts to not parse format version file and detached directory [#21351](https://github.com/ClickHouse/ClickHouse/pull/21351) ([Fuwang Hu](https://github.com/fuwhu)). +* Persistent nukeeper snapshot storage [#21425](https://github.com/ClickHouse/ClickHouse/pull/21425) ([alesapin](https://github.com/alesapin)). +* Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) [#21436](https://github.com/ClickHouse/ClickHouse/pull/21436) ([Azat Khuzhin](https://github.com/azat)). +* Adjust prewhere_with_row_level_filter performance test [#21442](https://github.com/ClickHouse/ClickHouse/pull/21442) ([Denis Glazachev](https://github.com/traceon)). +* Refactor actions dag [#21459](https://github.com/ClickHouse/ClickHouse/pull/21459) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* add query formatting idempotence check to fuzzer [#21466](https://github.com/ClickHouse/ClickHouse/pull/21466) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix heap-buffer-overflow in highlighting multi-line comments [#21492](https://github.com/ClickHouse/ClickHouse/pull/21492) ([Azat Khuzhin](https://github.com/azat)). +* Fix global stop merges in test [#21508](https://github.com/ClickHouse/ClickHouse/pull/21508) ([alesapin](https://github.com/alesapin)). +* DirectDictionary updated [#21513](https://github.com/ClickHouse/ClickHouse/pull/21513) ([Maksim Kita](https://github.com/kitaisreal)). +* Avoid processing optimize_skip_unused_shards twice [#21526](https://github.com/ClickHouse/ClickHouse/pull/21526) ([Azat Khuzhin](https://github.com/azat)). +* DOCSUP-7197: Escaped Unicode replaced with symbols [#21530](https://github.com/ClickHouse/ClickHouse/pull/21530) ([lehasm](https://github.com/lehasm)). +* Pod array left pad not multiple of element crash fix [#21532](https://github.com/ClickHouse/ClickHouse/pull/21532) ([Maksim Kita](https://github.com/kitaisreal)). +* ShellCommand waitpid eintr signal fix [#21546](https://github.com/ClickHouse/ClickHouse/pull/21546) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactoring of data types serialization [#21562](https://github.com/ClickHouse/ClickHouse/pull/21562) ([Anton Popov](https://github.com/CurtizJ)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added specialized CacheDictionaryStorage [#21573](https://github.com/ClickHouse/ClickHouse/pull/21573) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Union merge for arcadia_skip_list.txt to avoid frequent conflicts [#21580](https://github.com/ClickHouse/ClickHouse/pull/21580) ([Azat Khuzhin](https://github.com/azat)). +* Enable ipv6 in NuRaft [#21593](https://github.com/ClickHouse/ClickHouse/pull/21593) ([alesapin](https://github.com/alesapin)). +* add an article about ast-based fuzzer [#21608](https://github.com/ClickHouse/ClickHouse/pull/21608) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.3 [#21618](https://github.com/ClickHouse/ClickHouse/pull/21618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* redefine some errorcode [#21629](https://github.com/ClickHouse/ClickHouse/pull/21629) ([flynn](https://github.com/ucasfl)). +* Fix ambigous column error in joins_in_memory [#21658](https://github.com/ClickHouse/ClickHouse/pull/21658) ([Vladimir C](https://github.com/vdimir)). +* Add test for path as a query parameter in system.zookeeper [#21661](https://github.com/ClickHouse/ClickHouse/pull/21661) ([Kruglov Pavel](https://github.com/Avogar)). +* ExecutablePool fix default max execution time setting [#21662](https://github.com/ClickHouse/ClickHouse/pull/21662) ([Maksim Kita](https://github.com/kitaisreal)). +* DictionaryStructure fix non unique attribute names [#21668](https://github.com/ClickHouse/ClickHouse/pull/21668) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky test_replace_partition [#21674](https://github.com/ClickHouse/ClickHouse/pull/21674) ([Azat Khuzhin](https://github.com/azat)). +* DOC Fix ORDER BY syntax [#21675](https://github.com/ClickHouse/ClickHouse/pull/21675) ([Michael Monashev](https://github.com/MichaelMonashev)). +* PODArray swap fix [#21678](https://github.com/ClickHouse/ClickHouse/pull/21678) ([Maksim Kita](https://github.com/kitaisreal)). +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). +* Run three nodes with Replicated database and NuKeeper in CI [#21690](https://github.com/ClickHouse/ClickHouse/pull/21690) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error message in clickhouse-test [#21691](https://github.com/ClickHouse/ClickHouse/pull/21691) ([Azat Khuzhin](https://github.com/azat)). +* Set SOCK_CLOEXEC for sockets (hardcoded via poco update) [#21695](https://github.com/ClickHouse/ClickHouse/pull/21695) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes (that was found by stress tests) [#21696](https://github.com/ClickHouse/ClickHouse/pull/21696) ([Azat Khuzhin](https://github.com/azat)). +* Fix log_comment for *.sh in clickhouse-test [#21700](https://github.com/ClickHouse/ClickHouse/pull/21700) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless CMake option [#21712](https://github.com/ClickHouse/ClickHouse/pull/21712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in modulo by constant [#21713](https://github.com/ClickHouse/ClickHouse/pull/21713) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more variants for memcpy benchmark [#21715](https://github.com/ClickHouse/ClickHouse/pull/21715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests [#21724](https://github.com/ClickHouse/ClickHouse/pull/21724) ([Azat Khuzhin](https://github.com/azat)). +* make the fuzzer use sources from the CI [#21754](https://github.com/ClickHouse/ClickHouse/pull/21754) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add one more variant to memcpy benchmark [#21759](https://github.com/ClickHouse/ClickHouse/pull/21759) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* docs(fix): typo [#21775](https://github.com/ClickHouse/ClickHouse/pull/21775) ([Ali Demirci](https://github.com/depyronick)). +* DDLWorker.cpp: fixed exceeded amount of tries typo [#21807](https://github.com/ClickHouse/ClickHouse/pull/21807) ([Eldar Nasyrov](https://github.com/3ldar-nasyrov)). +* fix integration MaterializeMySQL test [#21819](https://github.com/ClickHouse/ClickHouse/pull/21819) ([TCeason](https://github.com/TCeason)). +* more robust error handling in perf test [#21846](https://github.com/ClickHouse/ClickHouse/pull/21846) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#17302](https://github.com/ClickHouse/ClickHouse/issues/17302) [#21848](https://github.com/ClickHouse/ClickHouse/pull/21848) ([Denny Crane](https://github.com/den-crane)). +* Add bash completion support for clickhouse utils [#21853](https://github.com/ClickHouse/ClickHouse/pull/21853) ([Azat Khuzhin](https://github.com/azat)). +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* fix fuzzer failure in tupleElement formatting [#21896](https://github.com/ClickHouse/ClickHouse/pull/21896) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix possibly dangling reference to Context [#21913](https://github.com/ClickHouse/ClickHouse/pull/21913) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add stress test for distributed queries [#21944](https://github.com/ClickHouse/ClickHouse/pull/21944) ([Azat Khuzhin](https://github.com/azat)). +* Fix misleading log in WriteBufferFromS3 [#21954](https://github.com/ClickHouse/ClickHouse/pull/21954) ([flynn](https://github.com/ucasfl)). +* Add a test for [#21991](https://github.com/ClickHouse/ClickHouse/issues/21991) [#21995](https://github.com/ClickHouse/ClickHouse/pull/21995) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11720](https://github.com/ClickHouse/ClickHouse/issues/11720) [#21997](https://github.com/ClickHouse/ClickHouse/pull/21997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15784](https://github.com/ClickHouse/ClickHouse/issues/15784) [#22002](https://github.com/ClickHouse/ClickHouse/pull/22002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* prevent accidental reinterpret_cast in Field::get<> [#22003](https://github.com/ClickHouse/ClickHouse/pull/22003) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in addMonths [#22006](https://github.com/ClickHouse/ClickHouse/pull/22006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#7963](https://github.com/ClickHouse/ClickHouse/issues/7963) [#22007](https://github.com/ClickHouse/ClickHouse/pull/22007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in intDiv [#21769](https://github.com/ClickHouse/ClickHouse/issues/21769) [#22009](https://github.com/ClickHouse/ClickHouse/pull/22009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cast to enum nullable fix [#22026](https://github.com/ClickHouse/ClickHouse/pull/22026) ([Maksim Kita](https://github.com/kitaisreal)). +* Small simplification in ExternalLoader. [#22027](https://github.com/ClickHouse/ClickHouse/pull/22027) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for [#21760](https://github.com/ClickHouse/ClickHouse/issues/21760) [#22036](https://github.com/ClickHouse/ClickHouse/pull/22036) ([Anton Popov](https://github.com/CurtizJ)). +* quick fix for broken resolution of apt.llvm.org on Yandex infra [#22055](https://github.com/ClickHouse/ClickHouse/pull/22055) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Darwin cmake disable memcpy benchmark [#22056](https://github.com/ClickHouse/ClickHouse/pull/22056) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix UBSan report in TransformDateTime64 [#22062](https://github.com/ClickHouse/ClickHouse/pull/22062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in mapPopulateSeries. [#22099](https://github.com/ClickHouse/ClickHouse/pull/22099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bar with invalid float value [#22114](https://github.com/ClickHouse/ClickHouse/pull/22114) ([flynn](https://github.com/ucasfl)). +* remove useless code [#22117](https://github.com/ClickHouse/ClickHouse/pull/22117) ([flynn](https://github.com/ucasfl)). +* stable formatting for negate() [#22133](https://github.com/ClickHouse/ClickHouse/pull/22133) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* adjust perf test thresholds [#22148](https://github.com/ClickHouse/ClickHouse/pull/22148) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix sleep_in_send_tables_status_ms/sleep_in_send_data_ms in integration tests [#22151](https://github.com/ClickHouse/ClickHouse/pull/22151) ([Azat Khuzhin](https://github.com/azat)). +* Update requirements.txt [#22153](https://github.com/ClickHouse/ClickHouse/pull/22153) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some flaky order dependent integration tests. [#22170](https://github.com/ClickHouse/ClickHouse/pull/22170) ([alesapin](https://github.com/alesapin)). +* Prevent busy waiting in hedged requests when async_socket_for_remote=0 [#22172](https://github.com/ClickHouse/ClickHouse/pull/22172) ([Kruglov Pavel](https://github.com/Avogar)). +* less flaky functional tests [#22181](https://github.com/ClickHouse/ClickHouse/pull/22181) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489) [#22219](https://github.com/ClickHouse/ClickHouse/pull/22219) ([Denny Crane](https://github.com/den-crane)). +* CachedCompressedReadBuffer fix cache usage [#22225](https://github.com/ClickHouse/ClickHouse/pull/22225) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix MSan report in `quantileDeterministic` [#22235](https://github.com/ClickHouse/ClickHouse/pull/22235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove harmful default parameters [#22238](https://github.com/ClickHouse/ClickHouse/pull/22238) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build error [#22243](https://github.com/ClickHouse/ClickHouse/pull/22243) ([hexiaoting](https://github.com/hexiaoting)). +* Rename NuKeeper and TestKeeper to Keeper in all places [#22274](https://github.com/ClickHouse/ClickHouse/pull/22274) ([alesapin](https://github.com/alesapin)). +* Update materialize-mysql.md [#22275](https://github.com/ClickHouse/ClickHouse/pull/22275) ([曲正鹏](https://github.com/quzhengpeng)). +* Fix native macOS build for ALL_BUILD (Xcode/AppleClang) [#22289](https://github.com/ClickHouse/ClickHouse/pull/22289) ([Denis Glazachev](https://github.com/traceon)). +* Add suffixes for dockerfile arguments [#22301](https://github.com/ClickHouse/ClickHouse/pull/22301) ([filimonov](https://github.com/filimonov)). +* More coarse test for DateLUT [#22320](https://github.com/ClickHouse/ClickHouse/pull/22320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#22328](https://github.com/ClickHouse/ClickHouse/pull/22328) ([Anton Popov](https://github.com/CurtizJ)). +* Maybe fix false MSan report in GRPC [#22338](https://github.com/ClickHouse/ClickHouse/pull/22338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 3) [#22357](https://github.com/ClickHouse/ClickHouse/pull/22357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 4) [#22358](https://github.com/ClickHouse/ClickHouse/pull/22358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky tests test_row_policy* and test_quota* [#22371](https://github.com/ClickHouse/ClickHouse/pull/22371) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix flaky rabbitmq test [#22380](https://github.com/ClickHouse/ClickHouse/pull/22380) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix UBSan report in mapOp [#22389](https://github.com/ClickHouse/ClickHouse/pull/22389) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove recursive submodules from Arrow [#22390](https://github.com/ClickHouse/ClickHouse/pull/22390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some OOMs in stress tests [#22396](https://github.com/ClickHouse/ClickHouse/pull/22396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly place debug helpers [#22407](https://github.com/ClickHouse/ClickHouse/pull/22407) ([Amos Bird](https://github.com/amosbird)). +* fix error message for invalid window frame start [#22412](https://github.com/ClickHouse/ClickHouse/pull/22412) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Lower scale of a test [#22446](https://github.com/ClickHouse/ClickHouse/pull/22446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TestFlows (2) [#22480](https://github.com/ClickHouse/ClickHouse/pull/22480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature (datasketches support in clickhouse #14893) * Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.4.3.21-stable.md b/docs/changelogs/v21.4.3.21-stable.md index dc3d7b7005b..a320e16f06a 100644 --- a/docs/changelogs/v21.4.3.21-stable.md +++ b/docs/changelogs/v21.4.3.21-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.3.21-stable FIXME as compared to v21.4.2.10-prestable #### Improvement @@ -16,3 +23,7 @@ * Backported in [#22921](https://github.com/ClickHouse/ClickHouse/issues/22921): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Backported in [#22957](https://github.com/ClickHouse/ClickHouse/issues/22957): Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.4.30-stable.md b/docs/changelogs/v21.4.4.30-stable.md index f029d334fc7..755bbede8f3 100644 --- a/docs/changelogs/v21.4.4.30-stable.md +++ b/docs/changelogs/v21.4.4.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.4.30-stable FIXME as compared to v21.4.3.21-stable #### Backward Incompatible Change diff --git a/docs/changelogs/v21.4.5.46-stable.md b/docs/changelogs/v21.4.5.46-stable.md index 664037ba596..9ca0553a154 100644 --- a/docs/changelogs/v21.4.5.46-stable.md +++ b/docs/changelogs/v21.4.5.46-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.5.46-stable FIXME as compared to v21.4.4.30-stable #### Improvement @@ -19,3 +26,9 @@ * Backported in [#23534](https://github.com/ClickHouse/ClickHouse/issues/23534): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). * Backported in [#23531](https://github.com/ClickHouse/ClickHouse/issues/23531): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.6.55-stable.md b/docs/changelogs/v21.4.6.55-stable.md index ea3e413ea0c..daca28d596c 100644 --- a/docs/changelogs/v21.4.6.55-stable.md +++ b/docs/changelogs/v21.4.6.55-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.6.55-stable FIXME as compared to v21.4.5.46-stable #### Improvement diff --git a/docs/changelogs/v21.4.7.3-stable.md b/docs/changelogs/v21.4.7.3-stable.md index 0dad6cfcb2b..612201a73ee 100644 --- a/docs/changelogs/v21.4.7.3-stable.md +++ b/docs/changelogs/v21.4.7.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.7.3-stable FIXME as compared to v21.4.6.55-stable #### Bug Fix @@ -15,3 +22,7 @@ * Backported in [#24215](https://github.com/ClickHouse/ClickHouse/issues/24215): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#24243](https://github.com/ClickHouse/ClickHouse/issues/24243): Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.1.6601-prestable.md b/docs/changelogs/v21.5.1.6601-prestable.md index d64936fefce..dbe75de93d4 100644 --- a/docs/changelogs/v21.5.1.6601-prestable.md +++ b/docs/changelogs/v21.5.1.6601-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.1.6601-prestable FIXME as compared to v21.4.1.6422-prestable #### Backward Incompatible Change @@ -34,7 +41,7 @@ * Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)). * Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)). * Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)). -* MaterializeMySQL (experimental feature). Make ClickHouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). +* MaterializeMySQL (experimental feature). Make Clickhouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). * `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) ... [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)). * Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)). @@ -107,3 +114,96 @@ * NO CL ENTRY: 'Error message reads better'. [#22983](https://github.com/ClickHouse/ClickHouse/pull/22983) ([Igor O'sten](https://github.com/borodark)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix SIGSEGV by waiting servers thread pool [#21318](https://github.com/ClickHouse/ClickHouse/pull/21318) ([Azat Khuzhin](https://github.com/azat)). +* Better filter push down [#22087](https://github.com/ClickHouse/ClickHouse/pull/22087) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better tests for finalize in nested writers [#22110](https://github.com/ClickHouse/ClickHouse/pull/22110) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Replace all Context references with std::weak_ptr [#22297](https://github.com/ClickHouse/ClickHouse/pull/22297) ([Ivan](https://github.com/abyss7)). +* make some perf test queries more stable [#22324](https://github.com/ClickHouse/ClickHouse/pull/22324) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix ccache broken by prlimit [#22356](https://github.com/ClickHouse/ClickHouse/pull/22356) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove old MSan suppressions (part 5) [#22359](https://github.com/ClickHouse/ClickHouse/pull/22359) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for copier [#22441](https://github.com/ClickHouse/ClickHouse/pull/22441) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix window frame offset check and add more tests [#22459](https://github.com/ClickHouse/ClickHouse/pull/22459) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* FormatSettings null_as_default default value fix [#22528](https://github.com/ClickHouse/ClickHouse/pull/22528) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor fixes in tests for AArch64 [#22534](https://github.com/ClickHouse/ClickHouse/pull/22534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Filter removed/renamed tests from ci-changed-files.txt for fuzzer [#22542](https://github.com/ClickHouse/ClickHouse/pull/22542) ([Azat Khuzhin](https://github.com/azat)). +* Try fix flaky test [#22543](https://github.com/ClickHouse/ClickHouse/pull/22543) ([Alexander Tokmakov](https://github.com/tavplubix)). +* AppleClang compilation fix [#22561](https://github.com/ClickHouse/ClickHouse/pull/22561) ([Denis Glazachev](https://github.com/traceon)). +* Fix assert in Arena when doing GROUP BY Array of Nothing of non-zero size. [#22565](https://github.com/ClickHouse/ClickHouse/pull/22565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor improvement in index deserialization [#22586](https://github.com/ClickHouse/ClickHouse/pull/22586) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test after [#22427](https://github.com/ClickHouse/ClickHouse/issues/22427) [#22587](https://github.com/ClickHouse/ClickHouse/pull/22587) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix comments [#22589](https://github.com/ClickHouse/ClickHouse/pull/22589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some uncaught exceptions (in SCOPE_EXIT) under memory pressure [#22592](https://github.com/ClickHouse/ClickHouse/pull/22592) ([Azat Khuzhin](https://github.com/azat)). +* Introduce IStorage::distributedWrite method for distributed INSERT SELECTS [#22593](https://github.com/ClickHouse/ClickHouse/pull/22593) ([Max Akhmedov](https://github.com/zlobober)). +* Fix flaky test 00816_long_concurrent_alter_column [#22628](https://github.com/ClickHouse/ClickHouse/pull/22628) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FlatDictionary fix perf test [#22629](https://github.com/ClickHouse/ClickHouse/pull/22629) ([Maksim Kita](https://github.com/kitaisreal)). +* DirectDictionary dictGet multiple columns optimization [#22630](https://github.com/ClickHouse/ClickHouse/pull/22630) ([Maksim Kita](https://github.com/kitaisreal)). +* Better retries on ZK errors in sh tests [#22633](https://github.com/ClickHouse/ClickHouse/pull/22633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add log_comment setting for DROP/CREATE DATABASE in clickhouse-test [#22646](https://github.com/ClickHouse/ClickHouse/pull/22646) ([Azat Khuzhin](https://github.com/azat)). +* Add retires for docker-compose pull in integration tests [#22647](https://github.com/ClickHouse/ClickHouse/pull/22647) ([Azat Khuzhin](https://github.com/azat)). +* Fix impossible invalid-read for system.errors accounting [#22655](https://github.com/ClickHouse/ClickHouse/pull/22655) ([Azat Khuzhin](https://github.com/azat)). +* Skip compiling xz if we're using system xz (unbundled) [#22656](https://github.com/ClickHouse/ClickHouse/pull/22656) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix test 01294_create_settings_profile [#22662](https://github.com/ClickHouse/ClickHouse/pull/22662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix test 01039_row_policy_dcl [#22663](https://github.com/ClickHouse/ClickHouse/pull/22663) ([Anton Popov](https://github.com/CurtizJ)). +* Another attempt to enable pytest [#22664](https://github.com/ClickHouse/ClickHouse/pull/22664) ([Ivan](https://github.com/abyss7)). +* Fix random failures of tests that are using query_log [#22666](https://github.com/ClickHouse/ClickHouse/pull/22666) ([Anton Popov](https://github.com/CurtizJ)). +* fix build error 'always_inline' function might not be inlinable [#22667](https://github.com/ClickHouse/ClickHouse/pull/22667) ([flynn](https://github.com/ucasfl)). +* Add bool type in postgres engine [#22668](https://github.com/ClickHouse/ClickHouse/pull/22668) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix mutation killers tests [#22670](https://github.com/ClickHouse/ClickHouse/pull/22670) ([alesapin](https://github.com/alesapin)). +* Change Aggregatingmergetree to AggregatingMergeTree in docs [#22687](https://github.com/ClickHouse/ClickHouse/pull/22687) ([Kruglov Pavel](https://github.com/Avogar)). +* fix window functions with multiple input streams and no sorting [#22704](https://github.com/ClickHouse/ClickHouse/pull/22704) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove redundant fsync on coordination logs rotation [#22707](https://github.com/ClickHouse/ClickHouse/pull/22707) ([alesapin](https://github.com/alesapin)). +* Fix test 01702_system_query_log [#22708](https://github.com/ClickHouse/ClickHouse/pull/22708) ([Anton Popov](https://github.com/CurtizJ)). +* MemoryStorage sync comments and code [#22721](https://github.com/ClickHouse/ClickHouse/pull/22721) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix potential segfault on Keeper startup [#22743](https://github.com/ClickHouse/ClickHouse/pull/22743) ([alesapin](https://github.com/alesapin)). +* Avoid using harmful function rand() [#22744](https://github.com/ClickHouse/ClickHouse/pull/22744) ([Amos Bird](https://github.com/amosbird)). +* Fix flacky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)). +* add more messages when flushing the logs [#22761](https://github.com/ClickHouse/ClickHouse/pull/22761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Moved BorrowedObjectPool to common [#22764](https://github.com/ClickHouse/ClickHouse/pull/22764) ([Maksim Kita](https://github.com/kitaisreal)). +* Functions ExternalDictionaries standardize exception throw [#22821](https://github.com/ClickHouse/ClickHouse/pull/22821) ([Maksim Kita](https://github.com/kitaisreal)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). +* Small change in replicated database tests run [#22826](https://github.com/ClickHouse/ClickHouse/pull/22826) ([alesapin](https://github.com/alesapin)). +* Slightly improve logging messages for Distributed async sends [#22829](https://github.com/ClickHouse/ClickHouse/pull/22829) ([Azat Khuzhin](https://github.com/azat)). +* Fix what looks like a trivial mistake [#22833](https://github.com/ClickHouse/ClickHouse/pull/22833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for already fixed issue [#22855](https://github.com/ClickHouse/ClickHouse/pull/22855) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DataTypeLowCardinality format tsv parsing issue [#22863](https://github.com/ClickHouse/ClickHouse/pull/22863) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated MariaDB connector fix cmake [#22865](https://github.com/ClickHouse/ClickHouse/pull/22865) ([Maksim Kita](https://github.com/kitaisreal)). +* Prettify logs of integration tests [#22868](https://github.com/ClickHouse/ClickHouse/pull/22868) ([Anton Popov](https://github.com/CurtizJ)). +* Util `memcpy-bench` is built only when position independent code is disabled [#22875](https://github.com/ClickHouse/ClickHouse/pull/22875) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix vanilla GCC compilation in macOS [#22885](https://github.com/ClickHouse/ClickHouse/pull/22885) ([Denis Glazachev](https://github.com/traceon)). +* Better diagnostics for OOM in stress test [#22894](https://github.com/ClickHouse/ClickHouse/pull/22894) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionaries updated performance tests [#22925](https://github.com/ClickHouse/ClickHouse/pull/22925) ([Maksim Kita](https://github.com/kitaisreal)). +* IAggreagteFunction allocatesMemoryInArena removed default implementation [#22938](https://github.com/ClickHouse/ClickHouse/pull/22938) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flapping test_merge_tree_s3 test [#22942](https://github.com/ClickHouse/ClickHouse/pull/22942) ([ianton-ru](https://github.com/ianton-ru)). +* less flaky test [#22944](https://github.com/ClickHouse/ClickHouse/pull/22944) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Dictionaries standardize exceptions [#22961](https://github.com/ClickHouse/ClickHouse/pull/22961) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExternalDistributed arcadia fix [#22962](https://github.com/ClickHouse/ClickHouse/pull/22962) ([Maksim Kita](https://github.com/kitaisreal)). +* Check out of range values in FieldVisitorConverToNumber [#22964](https://github.com/ClickHouse/ClickHouse/pull/22964) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix combinators with common prefix name (State and SimpleState) with libstdc++ [#22977](https://github.com/ClickHouse/ClickHouse/pull/22977) ([Azat Khuzhin](https://github.com/azat)). +* Fix arcadia [#22982](https://github.com/ClickHouse/ClickHouse/pull/22982) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix excessive warning in StorageDistributed with cross-replication [#22990](https://github.com/ClickHouse/ClickHouse/pull/22990) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update MergeTreeData.cpp Better error message. [#22994](https://github.com/ClickHouse/ClickHouse/pull/22994) ([Denny Crane](https://github.com/den-crane)). +* Fix assertion when filtering tables in StorageMerge [#22998](https://github.com/ClickHouse/ClickHouse/pull/22998) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a test for [#2719](https://github.com/ClickHouse/ClickHouse/issues/2719) [#23008](https://github.com/ClickHouse/ClickHouse/pull/23008) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix multi response in TestKeeper [#23041](https://github.com/ClickHouse/ClickHouse/pull/23041) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve hung check in Stress tests [#23043](https://github.com/ClickHouse/ClickHouse/pull/23043) ([Alexander Tokmakov](https://github.com/tavplubix)). +* blog article about code review [#23045](https://github.com/ClickHouse/ClickHouse/pull/23045) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* LibraryDictionary bridge library interface [#23048](https://github.com/ClickHouse/ClickHouse/pull/23048) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove useless files [#23049](https://github.com/ClickHouse/ClickHouse/pull/23049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Upload keeper logs from stateless tests [#23077](https://github.com/ClickHouse/ClickHouse/pull/23077) ([alesapin](https://github.com/alesapin)). +* CI runner intergation tests logs update to tar.gz [#23078](https://github.com/ClickHouse/ClickHouse/pull/23078) ([Maksim Kita](https://github.com/kitaisreal)). +* Tiny logging improvements [#23083](https://github.com/ClickHouse/ClickHouse/pull/23083) ([Azat Khuzhin](https://github.com/azat)). +* Block all memory tracking limits in dtors/SCOPE_EXIT_*SAFE/tryLogCurrentException [#23086](https://github.com/ClickHouse/ClickHouse/pull/23086) ([Azat Khuzhin](https://github.com/azat)). +* jemalloc tuning [#23088](https://github.com/ClickHouse/ClickHouse/pull/23088) ([Azat Khuzhin](https://github.com/azat)). +* Rename strange tests [#23111](https://github.com/ClickHouse/ClickHouse/pull/23111) ([alesapin](https://github.com/alesapin)). +* Fix arcadia build S3 [#23114](https://github.com/ClickHouse/ClickHouse/pull/23114) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated zlib [#23153](https://github.com/ClickHouse/ClickHouse/pull/23153) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Change logging from trace to debug for messages with rows/bytes [#23160](https://github.com/ClickHouse/ClickHouse/pull/23160) ([Azat Khuzhin](https://github.com/azat)). +* More verbose logs for debuging test failures with Replicated and Keeper [#23161](https://github.com/ClickHouse/ClickHouse/pull/23161) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix exception message for "parts_to_throw_insert" [#23177](https://github.com/ClickHouse/ClickHouse/pull/23177) ([madianjun](https://github.com/mdianjun)). +* Fix flapping tests test_s3_zero_copy_replication [#23184](https://github.com/ClickHouse/ClickHouse/pull/23184) ([ianton-ru](https://github.com/ianton-ru)). +* Add a test for [#14610](https://github.com/ClickHouse/ClickHouse/issues/14610) [#23209](https://github.com/ClickHouse/ClickHouse/pull/23209) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for fixed issue [#9363](https://github.com/ClickHouse/ClickHouse/issues/9363) [#23216](https://github.com/ClickHouse/ClickHouse/pull/23216) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable postgresql_port in perf tests [#23218](https://github.com/ClickHouse/ClickHouse/pull/23218) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.5.2.25-prestable.md b/docs/changelogs/v21.5.2.25-prestable.md index 45e784218da..1d7418a30c2 100644 --- a/docs/changelogs/v21.5.2.25-prestable.md +++ b/docs/changelogs/v21.5.2.25-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.2.25-prestable FIXME as compared to v21.5.1.6601-prestable #### Improvement @@ -38,3 +45,10 @@ * Backported in [#23347](https://github.com/ClickHouse/ClickHouse/issues/23347):. [#23334](https://github.com/ClickHouse/ClickHouse/pull/23334) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Zlib use attribute constructor for functable initialization [#23266](https://github.com/ClickHouse/ClickHouse/pull/23266) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.3.1-prestable.md b/docs/changelogs/v21.5.3.1-prestable.md index 24b29d1495e..2e52d8ba6a8 100644 --- a/docs/changelogs/v21.5.3.1-prestable.md +++ b/docs/changelogs/v21.5.3.1-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.3.1-prestable FIXME as compared to v21.5.2.25-prestable #### Bug Fix @@ -7,3 +14,7 @@ * Backported in [#23832](https://github.com/ClickHouse/ClickHouse/issues/23832): Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#23927](https://github.com/ClickHouse/ClickHouse/issues/23927): HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.4.6-prestable.md b/docs/changelogs/v21.5.4.6-prestable.md index c9e040309cc..37de3225996 100644 --- a/docs/changelogs/v21.5.4.6-prestable.md +++ b/docs/changelogs/v21.5.4.6-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.4.6-prestable FIXME as compared to v21.5.3.1-prestable #### Bug Fix diff --git a/docs/changelogs/v21.5.5.12-stable.md b/docs/changelogs/v21.5.5.12-stable.md index b49e2c60b08..71467dd2ffa 100644 --- a/docs/changelogs/v21.5.5.12-stable.md +++ b/docs/changelogs/v21.5.5.12-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.5.12-stable FIXME as compared to v21.5.4.6-prestable #### Bug Fix diff --git a/docs/changelogs/v21.5.6.6-stable.md b/docs/changelogs/v21.5.6.6-stable.md index e6160dfa784..04fe3ec4010 100644 --- a/docs/changelogs/v21.5.6.6-stable.md +++ b/docs/changelogs/v21.5.6.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.6.6-stable FIXME as compared to v21.5.5.12-stable #### Bug Fix @@ -15,3 +22,9 @@ * Backported in [#24750](https://github.com/ClickHouse/ClickHouse/issues/24750):. [#23276](https://github.com/ClickHouse/ClickHouse/pull/23276) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.5.7.9-stable.md b/docs/changelogs/v21.5.7.9-stable.md index 9586e398547..eaf0cb6255b 100644 --- a/docs/changelogs/v21.5.7.9-stable.md +++ b/docs/changelogs/v21.5.7.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.7.9-stable FIXME as compared to v21.5.6.6-stable #### Improvement @@ -40,3 +47,9 @@ * NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.5: Remove endless `wait` from ZooKeeper client"'. [#24798](https://github.com/ClickHouse/ClickHouse/pull/24798) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.5.8.21-stable.md b/docs/changelogs/v21.5.8.21-stable.md index 4d9ffd687eb..fe59efd2255 100644 --- a/docs/changelogs/v21.5.8.21-stable.md +++ b/docs/changelogs/v21.5.8.21-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.8.21-stable FIXME as compared to v21.5.7.9-stable #### Bug Fix @@ -11,3 +18,8 @@ * Backported in [#25695](https://github.com/ClickHouse/ClickHouse/issues/25695): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#25754](https://github.com/ClickHouse/ClickHouse/issues/25754): Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backport unrelated changes, which fixes aliases bug [#25680](https://github.com/ClickHouse/ClickHouse/pull/25680) ([alesapin](https://github.com/alesapin)). +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.9.4-stable.md b/docs/changelogs/v21.5.9.4-stable.md index 17ef067194b..b9d643dda09 100644 --- a/docs/changelogs/v21.5.9.4-stable.md +++ b/docs/changelogs/v21.5.9.4-stable.md @@ -1,6 +1,18 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.9.4-stable FIXME as compared to v21.5.8.21-stable #### Bug Fix * Backported in [#25958](https://github.com/ClickHouse/ClickHouse/issues/25958): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). * Backported in [#26144](https://github.com/ClickHouse/ClickHouse/issues/26144): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.5 manual backport of [#25970](https://github.com/ClickHouse/ClickHouse/issues/25970) [#26139](https://github.com/ClickHouse/ClickHouse/pull/26139) ([Maksim Kita](https://github.com/kitaisreal)). +* 21.5 Docker unbundled fix [#26154](https://github.com/ClickHouse/ClickHouse/pull/26154) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.1.6891-prestable.md b/docs/changelogs/v21.6.1.6891-prestable.md index 7750cac32eb..11fcd854989 100644 --- a/docs/changelogs/v21.6.1.6891-prestable.md +++ b/docs/changelogs/v21.6.1.6891-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.1.6891-prestable FIXME as compared to v21.5.1.6601-prestable #### New Feature @@ -152,6 +159,169 @@ * NO CL ENTRY: 'Revert "Fix CI build for gcc-10"'. [#23772](https://github.com/ClickHouse/ClickHouse/pull/23772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Update syntax.md'. [#24267](https://github.com/ClickHouse/ClickHouse/pull/24267) ([lulichao](https://github.com/lulichao)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Apply idle_connnection_timeout/poll_interval after each query [#21938](https://github.com/ClickHouse/ClickHouse/pull/21938) ([Azat Khuzhin](https://github.com/azat)). +* Do not silently catch errors for writing to S3 [#22208](https://github.com/ClickHouse/ClickHouse/pull/22208) ([Azat Khuzhin](https://github.com/azat)). +* Add dockerhub-proxy to runner [#23138](https://github.com/ClickHouse/ClickHouse/pull/23138) ([Ilya Yatsishin](https://github.com/qoega)). +* merging sumCount fusion PR [#21337](https://github.com/ClickHouse/ClickHouse/issues/21337) [#23159](https://github.com/ClickHouse/ClickHouse/pull/23159) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Minor fixes in ATTACH query [#23189](https://github.com/ClickHouse/ClickHouse/pull/23189) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#22503](https://github.com/ClickHouse/ClickHouse/issues/22503) [#23195](https://github.com/ClickHouse/ClickHouse/pull/23195) ([Anton Popov](https://github.com/CurtizJ)). +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* more stable formatting for negate() [#23201](https://github.com/ClickHouse/ClickHouse/pull/23201) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Use double quote identifier in odbc as default in case of error [#23217](https://github.com/ClickHouse/ClickHouse/pull/23217) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add -Wundef for gcc builds [#23258](https://github.com/ClickHouse/ClickHouse/pull/23258) ([Azat Khuzhin](https://github.com/azat)). +* Report an error if jemalloc.background_thread was requested [#23259](https://github.com/ClickHouse/ClickHouse/pull/23259) ([Azat Khuzhin](https://github.com/azat)). +* Add trace_log into stateless/stress test artifacts [#23264](https://github.com/ClickHouse/ClickHouse/pull/23264) ([Azat Khuzhin](https://github.com/azat)). +* Zlib use attribute constructor for functable initialization [#23266](https://github.com/ClickHouse/ClickHouse/pull/23266) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function toDateTime decimal overflow ubsan fix [#23278](https://github.com/ClickHouse/ClickHouse/pull/23278) ([Maksim Kita](https://github.com/kitaisreal)). +* Link keeper-bench to clickhouse_common_zookeeper [#23302](https://github.com/ClickHouse/ClickHouse/pull/23302) ([Raúl Marín](https://github.com/Algunenano)). +* Print errors on db creation in clickhouse-test [#23304](https://github.com/ClickHouse/ClickHouse/pull/23304) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix broken perf test [#23308](https://github.com/ClickHouse/ClickHouse/pull/23308) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix bad test 01602_max_distributed_connections [#23317](https://github.com/ClickHouse/ClickHouse/pull/23317) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix hdfs reading from files with spaces [#23318](https://github.com/ClickHouse/ClickHouse/pull/23318) ([Kseniia Sumarokova](https://github.com/kssenii)). +* add check that p.pruning works [#23321](https://github.com/ClickHouse/ClickHouse/pull/23321) ([Denny Crane](https://github.com/den-crane)). +* Add test for [#7815](https://github.com/ClickHouse/ClickHouse/issues/7815) [#23332](https://github.com/ClickHouse/ClickHouse/pull/23332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01666_merge_tree_max_query_limit.sh [#23335](https://github.com/ClickHouse/ClickHouse/pull/23335) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#12077](https://github.com/ClickHouse/ClickHouse/issues/12077) [#23352](https://github.com/ClickHouse/ClickHouse/pull/23352) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable clickhouse-odbc-bridge build when ODBC is disabled [#23357](https://github.com/ClickHouse/ClickHouse/pull/23357) ([Denis Glazachev](https://github.com/traceon)). +* Fix AppleClang build [#23358](https://github.com/ClickHouse/ClickHouse/pull/23358) ([Denis Glazachev](https://github.com/traceon)). +* Use Atomic database for development environment [#23377](https://github.com/ClickHouse/ClickHouse/pull/23377) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#2582](https://github.com/ClickHouse/ClickHouse/issues/2582) [#23378](https://github.com/ClickHouse/ClickHouse/pull/23378) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#1647](https://github.com/ClickHouse/ClickHouse/issues/1647) [#23379](https://github.com/ClickHouse/ClickHouse/pull/23379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow rabbitmq vhost in table settings [#23452](https://github.com/ClickHouse/ClickHouse/pull/23452) ([Kseniia Sumarokova](https://github.com/kssenii)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). +* ISSUES-23310 Try fix MySQL 8.0 address already in use [#23462](https://github.com/ClickHouse/ClickHouse/pull/23462) ([Winter Zhang](https://github.com/zhang2014)). +* Fix error in perf test [#23469](https://github.com/ClickHouse/ClickHouse/pull/23469) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* clickhouse-test: print database name on failures [#23486](https://github.com/ClickHouse/ClickHouse/pull/23486) ([Azat Khuzhin](https://github.com/azat)). +* upload cpu model to perf test db [#23514](https://github.com/ClickHouse/ClickHouse/pull/23514) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix function tests flaps [#23517](https://github.com/ClickHouse/ClickHouse/pull/23517) ([Azat Khuzhin](https://github.com/azat)). +* fix pvs warnings [#23520](https://github.com/ClickHouse/ClickHouse/pull/23520) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ignore empty input chunks generated by joins [#23542](https://github.com/ClickHouse/ClickHouse/pull/23542) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix window functions for Distributed tables [#23546](https://github.com/ClickHouse/ClickHouse/pull/23546) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* add more info to perf test report [#23550](https://github.com/ClickHouse/ClickHouse/pull/23550) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* export trace log from stateless tests in flamegraph-friendly format [#23553](https://github.com/ClickHouse/ClickHouse/pull/23553) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* add numactl info to perf test run attributes [#23554](https://github.com/ClickHouse/ClickHouse/pull/23554) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Less chance of OOM in stress tests [#23561](https://github.com/ClickHouse/ClickHouse/pull/23561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix inconsistent formatting for tupleElement (for fuzzer) [#23595](https://github.com/ClickHouse/ClickHouse/pull/23595) ([Azat Khuzhin](https://github.com/azat)). +* Remove unneeded code from CMakeLists [#23597](https://github.com/ClickHouse/ClickHouse/pull/23597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Somewhat better handling of paths in CMake (incomplete) [#23598](https://github.com/ClickHouse/ClickHouse/pull/23598) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old trash (a little) [#23599](https://github.com/ClickHouse/ClickHouse/pull/23599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some garbage from RocksDB CMakeLists [#23601](https://github.com/ClickHouse/ClickHouse/pull/23601) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add warning about gcc usage [#23603](https://github.com/ClickHouse/ClickHouse/pull/23603) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove garbage from CMakeLists (2) [#23604](https://github.com/ClickHouse/ClickHouse/pull/23604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove rotten parts of release script [#23605](https://github.com/ClickHouse/ClickHouse/pull/23605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless file [#23606](https://github.com/ClickHouse/ClickHouse/pull/23606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Skip CatBoost tests under MSan [#23614](https://github.com/ClickHouse/ClickHouse/pull/23614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad test 01641_memory_tracking_insert_optimize_long [#23617](https://github.com/ClickHouse/ClickHouse/pull/23617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove pbuilder [#23618](https://github.com/ClickHouse/ClickHouse/pull/23618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Aggregator remove unused code [#23635](https://github.com/ClickHouse/ClickHouse/pull/23635) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#22984](https://github.com/ClickHouse/ClickHouse/issues/22984) [#23637](https://github.com/ClickHouse/ClickHouse/pull/23637) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Move non gtest unit tests to /examples folder [#23644](https://github.com/ClickHouse/ClickHouse/pull/23644) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Kerberized HDFS test is fluky - retries and extra output if failure [#23650](https://github.com/ClickHouse/ClickHouse/pull/23650) ([Ilya Golshtein](https://github.com/ilejn)). +* Fix documentation for DETACH ON CLUSTER PERMANENTLY [#23653](https://github.com/ClickHouse/ClickHouse/pull/23653) ([Azat Khuzhin](https://github.com/azat)). +* Skip integration test for library bridge under MSan [#23662](https://github.com/ClickHouse/ClickHouse/pull/23662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a few PVS-Studio warnings [#23663](https://github.com/ClickHouse/ClickHouse/pull/23663) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable use-after-destruction detection in MSan [#23664](https://github.com/ClickHouse/ClickHouse/pull/23664) ([Azat Khuzhin](https://github.com/azat)). +* Fix cyrus-sasl msan warning [#23672](https://github.com/ClickHouse/ClickHouse/pull/23672) ([Ilya Yatsishin](https://github.com/qoega)). +* A little bit faster merge of aggregating states. [#23681](https://github.com/ClickHouse/ClickHouse/pull/23681) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* stable query indexes in perf test [#23707](https://github.com/ClickHouse/ClickHouse/pull/23707) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Minor changes in code [#14254](https://github.com/ClickHouse/ClickHouse/issues/14254) [#23709](https://github.com/ClickHouse/ClickHouse/pull/23709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix illiterate error message [#23700](https://github.com/ClickHouse/ClickHouse/issues/23700) [#23710](https://github.com/ClickHouse/ClickHouse/pull/23710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete compilers [#23711](https://github.com/ClickHouse/ClickHouse/pull/23711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow-up for [#23644](https://github.com/ClickHouse/ClickHouse/issues/23644) [#23712](https://github.com/ClickHouse/ClickHouse/pull/23712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable test_odbc_interaction for MSan build [#23722](https://github.com/ClickHouse/ClickHouse/pull/23722) ([Vladimir C](https://github.com/vdimir)). +* fix a typo in query formatting check in fuzzer [#23726](https://github.com/ClickHouse/ClickHouse/pull/23726) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ExternalLoaderRepository fix arcadia [#23732](https://github.com/ClickHouse/ClickHouse/pull/23732) ([Maksim Kita](https://github.com/kitaisreal)). +* Ignore cmake-in-clickhouse [#23740](https://github.com/ClickHouse/ClickHouse/pull/23740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refactor join step [#23743](https://github.com/ClickHouse/ClickHouse/pull/23743) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* clickhouse-test: send TERM to all childs (to avoid hung check triggering) [#23750](https://github.com/ClickHouse/ClickHouse/pull/23750) ([Azat Khuzhin](https://github.com/azat)). +* Improve test_insert_into_distributed [#23751](https://github.com/ClickHouse/ClickHouse/pull/23751) ([Azat Khuzhin](https://github.com/azat)). +* Fix CI build for gcc-10 [#23760](https://github.com/ClickHouse/ClickHouse/pull/23760) ([Maksim Kita](https://github.com/kitaisreal)). +* Update array-functions.md [#23762](https://github.com/ClickHouse/ClickHouse/pull/23762) ([fancno](https://github.com/fancno)). +* Remove unused compilers (fixed for the troublesome "unbundled" build) [#23766](https://github.com/ClickHouse/ClickHouse/pull/23766) ([Maksim Kita](https://github.com/kitaisreal)). +* Slightly better hardening for intersecting parts [#23767](https://github.com/ClickHouse/ClickHouse/pull/23767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Workaround for PVS-Studio [#23776](https://github.com/ClickHouse/ClickHouse/pull/23776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add examples folder filter to ya.make.in [#23786](https://github.com/ClickHouse/ClickHouse/pull/23786) ([Maksim Kita](https://github.com/kitaisreal)). +* Function default implementation for nulls small optimization [#23799](https://github.com/ClickHouse/ClickHouse/pull/23799) ([Maksim Kita](https://github.com/kitaisreal)). +* autodetect arch of gosu in server dockerfile [#23802](https://github.com/ClickHouse/ClickHouse/pull/23802) ([filimonov](https://github.com/filimonov)). +* Add test for [#18170](https://github.com/ClickHouse/ClickHouse/issues/18170) [#23813](https://github.com/ClickHouse/ClickHouse/pull/23813) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* clickhouse-test: add missing whitespace before printing database on error [#23820](https://github.com/ClickHouse/ClickHouse/pull/23820) ([Azat Khuzhin](https://github.com/azat)). +* Improve 00840_long_concurrent_select_and_drop_deadlock [#23823](https://github.com/ClickHouse/ClickHouse/pull/23823) ([Azat Khuzhin](https://github.com/azat)). +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). +* XDBCBridgeHelper use global context [#23836](https://github.com/ClickHouse/ClickHouse/pull/23836) ([Maksim Kita](https://github.com/kitaisreal)). +* gcc-10 installation no-install-reccomends option fix [#23840](https://github.com/ClickHouse/ClickHouse/pull/23840) ([Maksim Kita](https://github.com/kitaisreal)). +* replxx readline compatibility [#23855](https://github.com/ClickHouse/ClickHouse/pull/23855) ([Azat Khuzhin](https://github.com/azat)). +* Add file paths into logs on failed distributed async sends [#23856](https://github.com/ClickHouse/ClickHouse/pull/23856) ([Azat Khuzhin](https://github.com/azat)). +* Reduce the amount of logs that StorageMergeTree::selectPartsToMutate outputs in busy systems. [#23863](https://github.com/ClickHouse/ClickHouse/pull/23863) ([Raúl Marín](https://github.com/Algunenano)). +* Add DiskRestartProxy.cpp to ya.make [#23868](https://github.com/ClickHouse/ClickHouse/pull/23868) ([Vladimir C](https://github.com/vdimir)). +* Fix some warnings by PVS-Studio [#23877](https://github.com/ClickHouse/ClickHouse/pull/23877) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Read/WriteBufferFromFileDecorator.cpp to ya.make [#23879](https://github.com/ClickHouse/ClickHouse/pull/23879) ([Vladimir C](https://github.com/vdimir)). +* @CurtizJ convinced me that this test has to be deleted [#23883](https://github.com/ClickHouse/ClickHouse/pull/23883) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* bash completion improvements [#23884](https://github.com/ClickHouse/ClickHouse/pull/23884) ([Azat Khuzhin](https://github.com/azat)). +* Remove obsolete code [#23886](https://github.com/ClickHouse/ClickHouse/pull/23886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code simplification (implemented TODO) [#23896](https://github.com/ClickHouse/ClickHouse/pull/23896) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add run options with default config path according to issues/23875 [#23898](https://github.com/ClickHouse/ClickHouse/pull/23898) ([ice1x](https://github.com/ice1x)). +* Fix test_insert_into_distributed flaps [#23903](https://github.com/ClickHouse/ClickHouse/pull/23903) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad test about compression codecs [#23908](https://github.com/ClickHouse/ClickHouse/pull/23908) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cleanup IDatabase.h from extra headers [#23912](https://github.com/ClickHouse/ClickHouse/pull/23912) ([Azat Khuzhin](https://github.com/azat)). +* Minor simplification [#23923](https://github.com/ClickHouse/ClickHouse/pull/23923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Drop unnecessary ports where applicable [#23928](https://github.com/ClickHouse/ClickHouse/pull/23928) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Check MIN/MAX attributes in the list of dictionary attributes [#23948](https://github.com/ClickHouse/ClickHouse/pull/23948) ([Azat Khuzhin](https://github.com/azat)). +* typo: fix a typo in Compression/CodecT64 [#23952](https://github.com/ClickHouse/ClickHouse/pull/23952) ([mwish](https://github.com/mapleFU)). +* Don't try GLIBC_COMPATIBILITY for i686 Linux [#23959](https://github.com/ClickHouse/ClickHouse/pull/23959) ([divanorama](https://github.com/divanorama)). +* Function arrayDifference decimal math overflow [#23961](https://github.com/ClickHouse/ClickHouse/pull/23961) ([Maksim Kita](https://github.com/kitaisreal)). +* Use 0 over nan for hit_rate in case of 0 queries to the cache dictionary [#23963](https://github.com/ClickHouse/ClickHouse/pull/23963) ([Azat Khuzhin](https://github.com/azat)). +* Round floats in Aggregator log messages [#23965](https://github.com/ClickHouse/ClickHouse/pull/23965) ([Azat Khuzhin](https://github.com/azat)). +* support longer query ids in trace log for perf tests [#23969](https://github.com/ClickHouse/ClickHouse/pull/23969) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* PVS-Studio fixes, part 6 [#23970](https://github.com/ClickHouse/ClickHouse/pull/23970) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Functional stateless tests fix numbers [#23974](https://github.com/ClickHouse/ClickHouse/pull/23974) ([Maksim Kita](https://github.com/kitaisreal)). +* use LowCardinality for AsynchronousMetricLog name column [#23981](https://github.com/ClickHouse/ClickHouse/pull/23981) ([flynn](https://github.com/ucasfl)). +* Function dictGetOrNull handle empty rows execute [#23990](https://github.com/ClickHouse/ClickHouse/pull/23990) ([Maksim Kita](https://github.com/kitaisreal)). +* CompileDAG fix Sip hash [#24004](https://github.com/ClickHouse/ClickHouse/pull/24004) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix bad code [#24007](https://github.com/ClickHouse/ClickHouse/pull/24007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better error codes in Keeper when no leader alive [#24017](https://github.com/ClickHouse/ClickHouse/pull/24017) ([alesapin](https://github.com/alesapin)). +* Fix ArenaWithFreeLists test [#24021](https://github.com/ClickHouse/ClickHouse/pull/24021) ([Maksim Kita](https://github.com/kitaisreal)). +* Run check_*_compiler_flag earlier [#24037](https://github.com/ClickHouse/ClickHouse/pull/24037) ([Amos Bird](https://github.com/amosbird)). +* Performance tests disable compile expressions [#24043](https://github.com/ClickHouse/ClickHouse/pull/24043) ([Maksim Kita](https://github.com/kitaisreal)). +* for trivial INSERT SELECT, adjust block size in bytes as well [#24048](https://github.com/ClickHouse/ClickHouse/pull/24048) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable thread_local in Arcadia build [#24051](https://github.com/ClickHouse/ClickHouse/pull/24051) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* lower two-level aggregation threshold for uniq test to avoid jitter [#24058](https://github.com/ClickHouse/ClickHouse/pull/24058) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix empty key projection query analysis [#24062](https://github.com/ClickHouse/ClickHouse/pull/24062) ([Amos Bird](https://github.com/amosbird)). +* Generate ya.make for missing UUID.cpp [#24064](https://github.com/ClickHouse/ClickHouse/pull/24064) ([Alexander Gololobov](https://github.com/davenger)). +* bash-completion: complete available formats [#24065](https://github.com/ClickHouse/ClickHouse/pull/24065) ([Azat Khuzhin](https://github.com/azat)). +* Fix concurrent snapshot read/write [#24073](https://github.com/ClickHouse/ClickHouse/pull/24073) ([alesapin](https://github.com/alesapin)). +* Also retry database creation in `clickhouse-test` [#24088](https://github.com/ClickHouse/ClickHouse/pull/24088) ([alesapin](https://github.com/alesapin)). +* Calculate header from ActionsDAG [#24108](https://github.com/ClickHouse/ClickHouse/pull/24108) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arcadia [#24133](https://github.com/ClickHouse/ClickHouse/pull/24133) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* TestReadAfterAIO: Use the current path instead of /tmp for temporal files [#24139](https://github.com/ClickHouse/ClickHouse/pull/24139) ([Raúl Marín](https://github.com/Algunenano)). +* Fix a few trailing whitespaces in output [#24150](https://github.com/ClickHouse/ClickHouse/pull/24150) ([Azat Khuzhin](https://github.com/azat)). +* IFunction refactoring [#24155](https://github.com/ClickHouse/ClickHouse/pull/24155) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix add projection to replicated mergetree [#24162](https://github.com/ClickHouse/ClickHouse/pull/24162) ([Amos Bird](https://github.com/amosbird)). +* Fix distributed processing when using projection [#24168](https://github.com/ClickHouse/ClickHouse/pull/24168) ([Amos Bird](https://github.com/amosbird)). +* fix tiny code style [#24169](https://github.com/ClickHouse/ClickHouse/pull/24169) ([flynn](https://github.com/ucasfl)). +* fix reinterpretAsFixedString for UUID [#24177](https://github.com/ClickHouse/ClickHouse/pull/24177) ([flynn](https://github.com/ucasfl)). +* Function move file [#24178](https://github.com/ClickHouse/ClickHouse/pull/24178) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated LRUHashMap benchmarks [#24182](https://github.com/ClickHouse/ClickHouse/pull/24182) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove temporary files [#24186](https://github.com/ClickHouse/ClickHouse/pull/24186) ([Kruglov Pavel](https://github.com/Avogar)). +* Try mute grpc msan [#24197](https://github.com/ClickHouse/ClickHouse/pull/24197) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update slack link [#24200](https://github.com/ClickHouse/ClickHouse/pull/24200) ([Ilya Yatsishin](https://github.com/qoega)). +* Simplier isLocalAddress [#24203](https://github.com/ClickHouse/ClickHouse/pull/24203) ([alesapin](https://github.com/alesapin)). +* Increase timeout for server restart in integration tests. [#24205](https://github.com/ClickHouse/ClickHouse/pull/24205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Addition to [#23997](https://github.com/ClickHouse/ClickHouse/issues/23997) [#24208](https://github.com/ClickHouse/ClickHouse/pull/24208) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Small code simplification [#24210](https://github.com/ClickHouse/ClickHouse/pull/24210) ([Maksim Kita](https://github.com/kitaisreal)). +* Speedup test [test_jbod_balancer/test.py] [#24247](https://github.com/ClickHouse/ClickHouse/pull/24247) ([alesapin](https://github.com/alesapin)). +* remove useless code [#24248](https://github.com/ClickHouse/ClickHouse/pull/24248) ([flynn](https://github.com/ucasfl)). +* One more attempt to fix retries in clickhouse-test [#24249](https://github.com/ClickHouse/ClickHouse/pull/24249) ([alesapin](https://github.com/alesapin)). +* Log exception in Allocator::free [#24256](https://github.com/ClickHouse/ClickHouse/pull/24256) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix broken HTML markup on website [#24265](https://github.com/ClickHouse/ClickHouse/pull/24265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix arcadia [#24282](https://github.com/ClickHouse/ClickHouse/pull/24282) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed Arcadia after IFunctionOverloadResolver interface refactoring [#24284](https://github.com/ClickHouse/ClickHouse/pull/24284) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix isLocalAddress() (ifa_addr maybe NULL) [#24308](https://github.com/ClickHouse/ClickHouse/pull/24308) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad test for Enum hints [#24313](https://github.com/ClickHouse/ClickHouse/pull/24313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature #14893 * - Add uniqThetaSketch to support [Theta Sketch](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html) in ClickHouse. [#22609](https://github.com/ClickHouse/ClickHouse/pull/22609) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.6.2.7-prestable.md b/docs/changelogs/v21.6.2.7-prestable.md index c5dec251786..55032e0258e 100644 --- a/docs/changelogs/v21.6.2.7-prestable.md +++ b/docs/changelogs/v21.6.2.7-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.2.7-prestable FIXME as compared to v21.6.1.6891-prestable #### Bug Fix @@ -11,3 +18,9 @@ * Backported in [#24597](https://github.com/ClickHouse/ClickHouse/issues/24597): Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#24600](https://github.com/ClickHouse/ClickHouse/issues/24600): Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). +* Fix several cases in cast operator [#24471](https://github.com/ClickHouse/ClickHouse/pull/24471) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.6.3.14-stable.md b/docs/changelogs/v21.6.3.14-stable.md index 8d2e2a03320..551f959aefd 100644 --- a/docs/changelogs/v21.6.3.14-stable.md +++ b/docs/changelogs/v21.6.3.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.3.14-stable FIXME as compared to v21.6.2.7-prestable #### Improvement diff --git a/docs/changelogs/v21.6.4.26-stable.md b/docs/changelogs/v21.6.4.26-stable.md index 9bf7d0e68cb..dc235d91e6d 100644 --- a/docs/changelogs/v21.6.4.26-stable.md +++ b/docs/changelogs/v21.6.4.26-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.4.26-stable FIXME as compared to v21.6.3.14-stable #### Bug Fix @@ -12,3 +19,8 @@ * Backported in [#25157](https://github.com/ClickHouse/ClickHouse/issues/25157): Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#25182](https://github.com/ClickHouse/ClickHouse/issues/25182): Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.5.37-stable.md b/docs/changelogs/v21.6.5.37-stable.md index 3d03c31c7e8..45a43199c28 100644 --- a/docs/changelogs/v21.6.5.37-stable.md +++ b/docs/changelogs/v21.6.5.37-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.5.37-stable FIXME as compared to v21.6.4.26-stable #### Improvement @@ -19,3 +26,8 @@ * Backported in [#25448](https://github.com/ClickHouse/ClickHouse/issues/25448): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#25407](https://github.com/ClickHouse/ClickHouse/issues/25407): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionActions compile only necessary places [#24273](https://github.com/ClickHouse/ClickHouse/pull/24273) ([Maksim Kita](https://github.com/kitaisreal)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.6.6.51-stable.md b/docs/changelogs/v21.6.6.51-stable.md index 55f1fd46119..39441adaa9e 100644 --- a/docs/changelogs/v21.6.6.51-stable.md +++ b/docs/changelogs/v21.6.6.51-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.6.51-stable FIXME as compared to v21.6.5.37-stable #### Bug Fix @@ -17,3 +24,7 @@ * NO CL ENTRY: 'Partial backport [#24061](https://github.com/ClickHouse/ClickHouse/issues/24061) to 21.6'. [#25621](https://github.com/ClickHouse/ClickHouse/pull/25621) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.7.57-stable.md b/docs/changelogs/v21.6.7.57-stable.md index 5ef9026794b..26887fac120 100644 --- a/docs/changelogs/v21.6.7.57-stable.md +++ b/docs/changelogs/v21.6.7.57-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.7.57-stable FIXME as compared to v21.6.6.51-stable #### Bug Fix @@ -6,3 +13,7 @@ * Backported in [#26096](https://github.com/ClickHouse/ClickHouse/issues/26096): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). * Backported in [#26143](https://github.com/ClickHouse/ClickHouse/issues/26143): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.6 manual backport of [#25970](https://github.com/ClickHouse/ClickHouse/issues/25970) [#26138](https://github.com/ClickHouse/ClickHouse/pull/26138) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.8.62-stable.md b/docs/changelogs/v21.6.8.62-stable.md index 1357c762181..0dc35d41344 100644 --- a/docs/changelogs/v21.6.8.62-stable.md +++ b/docs/changelogs/v21.6.8.62-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.8.62-stable FIXME as compared to v21.6.7.57-stable #### Bug Fix @@ -7,3 +14,7 @@ * Backported in [#26205](https://github.com/ClickHouse/ClickHouse/issues/26205): Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#26227](https://github.com/ClickHouse/ClickHouse/issues/26227): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Manual backport of [#25531](https://github.com/ClickHouse/ClickHouse/issues/25531) in 21.6 [#25776](https://github.com/ClickHouse/ClickHouse/pull/25776) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md index ecac9dd75ef..fae1c8d2fef 100644 --- a/docs/changelogs/v21.6.9.7-stable.md +++ b/docs/changelogs/v21.6.9.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.9.7-stable FIXME as compared to v21.6.8.62-stable #### Improvement @@ -41,3 +48,19 @@ * Backported in [#28264](https://github.com/ClickHouse/ClickHouse/issues/28264): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#28291](https://github.com/ClickHouse/ClickHouse/issues/28291): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.1.7283-prestable.md b/docs/changelogs/v21.7.1.7283-prestable.md index 52de493c4ab..a862e6c4750 100644 --- a/docs/changelogs/v21.7.1.7283-prestable.md +++ b/docs/changelogs/v21.7.1.7283-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.1.7283-prestable FIXME as compared to v21.6.1.6891-prestable #### Backward Incompatible Change @@ -172,6 +179,236 @@ * NO CL ENTRY: 'Revert "Add run-id option to integration tests"'. [#25526](https://github.com/ClickHouse/ClickHouse/pull/25526) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Revert "Implement h3ToGeo function"'. [#25593](https://github.com/ClickHouse/ClickHouse/pull/25593) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Part movement between shards [#17871](https://github.com/ClickHouse/ClickHouse/pull/17871) ([nvartolomei](https://github.com/nvartolomei)). +* Pass Settings to aggregate function creator [#22762](https://github.com/ClickHouse/ClickHouse/pull/22762) ([Vladimir C](https://github.com/vdimir)). +* Add uniqTheta in performance test [#23311](https://github.com/ClickHouse/ClickHouse/pull/23311) ([Kruglov Pavel](https://github.com/Avogar)). +* More pytest fixes [#23538](https://github.com/ClickHouse/ClickHouse/pull/23538) ([Ivan](https://github.com/abyss7)). +* Improved `test_storage_s3_get_unstable` [#23976](https://github.com/ClickHouse/ClickHouse/pull/23976) ([Vladimir Chebotarev](https://github.com/excitoon)). +* CompileExpressions comparison function constant case fix [#24023](https://github.com/ClickHouse/ClickHouse/pull/24023) ([Maksim Kita](https://github.com/kitaisreal)). +* Added llvm-project submodule [#24030](https://github.com/ClickHouse/ClickHouse/pull/24030) ([Maksim Kita](https://github.com/kitaisreal)). +* complain about unstable perf test queries [#24049](https://github.com/ClickHouse/ClickHouse/pull/24049) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.5 [#24098](https://github.com/ClickHouse/ClickHouse/pull/24098) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Implement antlr parser for projections [#24245](https://github.com/ClickHouse/ClickHouse/pull/24245) ([Amos Bird](https://github.com/amosbird)). +* Try to fix GROUP BY _shard_num in a different way [#24252](https://github.com/ClickHouse/ClickHouse/pull/24252) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* ExpressionActions compile only necessary places [#24273](https://github.com/ClickHouse/ClickHouse/pull/24273) ([Maksim Kita](https://github.com/kitaisreal)). +* consolidate connection loss handling in fuzzer [#24290](https://github.com/ClickHouse/ClickHouse/pull/24290) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Skip data finalization when doing projection materialization. [#24296](https://github.com/ClickHouse/ClickHouse/pull/24296) ([Amos Bird](https://github.com/amosbird)). +* One more error to retry in clickhouse-test [#24307](https://github.com/ClickHouse/ClickHouse/pull/24307) ([alesapin](https://github.com/alesapin)). +* Merging [#23461](https://github.com/ClickHouse/ClickHouse/issues/23461) [#24311](https://github.com/ClickHouse/ClickHouse/pull/24311) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove AutoArray [#24320](https://github.com/ClickHouse/ClickHouse/pull/24320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Compile expressions added partition by test [#24323](https://github.com/ClickHouse/ClickHouse/pull/24323) ([Maksim Kita](https://github.com/kitaisreal)). +* PODArray insert in the middle tests [#24333](https://github.com/ClickHouse/ClickHouse/pull/24333) ([Maksim Kita](https://github.com/kitaisreal)). +* remove retries from fast test [#24338](https://github.com/ClickHouse/ClickHouse/pull/24338) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Better parallelism for functional tests [#24349](https://github.com/ClickHouse/ClickHouse/pull/24349) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* AARCH64 hash tables benchmark [#24364](https://github.com/ClickHouse/ClickHouse/pull/24364) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix diagnostics in documentation script [#24379](https://github.com/ClickHouse/ClickHouse/pull/24379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 01033_quota_dcl [#24393](https://github.com/ClickHouse/ClickHouse/pull/24393) ([alesapin](https://github.com/alesapin)). +* Fix logical error AggregateFunctionFactory returned nullptr [#24398](https://github.com/ClickHouse/ClickHouse/pull/24398) ([Kruglov Pavel](https://github.com/Avogar)). +* Add log record for removed from AWS S3 keys [#24400](https://github.com/ClickHouse/ClickHouse/pull/24400) ([ianton-ru](https://github.com/ianton-ru)). +* Added AggregateFunctionSegmentLengthSum to ya.make [#24408](https://github.com/ClickHouse/ClickHouse/pull/24408) ([Maksim Kita](https://github.com/kitaisreal)). +* Stateless tests fixes [#24411](https://github.com/ClickHouse/ClickHouse/pull/24411) ([Azat Khuzhin](https://github.com/azat)). +* Function constant result with nonconstant arguments [#24417](https://github.com/ClickHouse/ClickHouse/pull/24417) ([Maksim Kita](https://github.com/kitaisreal)). +* Libunwind update version [#24419](https://github.com/ClickHouse/ClickHouse/pull/24419) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not built clickhouse-keeper w/o NuRaft [#24421](https://github.com/ClickHouse/ClickHouse/pull/24421) ([Azat Khuzhin](https://github.com/azat)). +* Switch message level to WARNING for FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION=OFF [#24423](https://github.com/ClickHouse/ClickHouse/pull/24423) ([Azat Khuzhin](https://github.com/azat)). +* Merging "experimental compression codecs" [#24424](https://github.com/ClickHouse/ClickHouse/pull/24424) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix formatting of negative values [#24427](https://github.com/ClickHouse/ClickHouse/pull/24427) ([Azat Khuzhin](https://github.com/azat)). +* Fix dictionary functions documentation [#24432](https://github.com/ClickHouse/ClickHouse/pull/24432) ([Maksim Kita](https://github.com/kitaisreal)). +* Mark false positives for PVS-Studio [#24434](https://github.com/ClickHouse/ClickHouse/pull/24434) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ActionsDAG compile expressions update [#24442](https://github.com/ClickHouse/ClickHouse/pull/24442) ([Maksim Kita](https://github.com/kitaisreal)). +* LLVM remove orc jit library [#24443](https://github.com/ClickHouse/ClickHouse/pull/24443) ([Maksim Kita](https://github.com/kitaisreal)). +* Add YAMLParser to ya.make [#24447](https://github.com/ClickHouse/ClickHouse/pull/24447) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). +* Fix fast test [#24454](https://github.com/ClickHouse/ClickHouse/pull/24454) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test 01085_max_distributed_connections [#24455](https://github.com/ClickHouse/ClickHouse/pull/24455) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Manually disable LLVM parts submodule update [#24460](https://github.com/ClickHouse/ClickHouse/pull/24460) ([Maksim Kita](https://github.com/kitaisreal)). +* Account total_rows_approx as soon as possible [#24462](https://github.com/ClickHouse/ClickHouse/pull/24462) ([Azat Khuzhin](https://github.com/azat)). +* Fix header mismatch for UNION. [#24463](https://github.com/ClickHouse/ClickHouse/pull/24463) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to improve kafka flaky test [#24465](https://github.com/ClickHouse/ClickHouse/pull/24465) ([filimonov](https://github.com/filimonov)). +* CompileExpression cached functions with context fix [#24468](https://github.com/ClickHouse/ClickHouse/pull/24468) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed IFunction then typos [#24469](https://github.com/ClickHouse/ClickHouse/pull/24469) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix several cases in cast operator [#24471](https://github.com/ClickHouse/ClickHouse/pull/24471) ([Anton Popov](https://github.com/CurtizJ)). +* MemoryTracker new no throw [#24483](https://github.com/ClickHouse/ClickHouse/pull/24483) ([Maksim Kita](https://github.com/kitaisreal)). +* Revent libunwind [#24485](https://github.com/ClickHouse/ClickHouse/pull/24485) ([Maksim Kita](https://github.com/kitaisreal)). +* Log file name on error in FileChecker ctor [#24489](https://github.com/ClickHouse/ClickHouse/pull/24489) ([Vladimir C](https://github.com/vdimir)). +* Remove containers in intergational runner entrypoint [#24492](https://github.com/ClickHouse/ClickHouse/pull/24492) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix benign race (detected by clang-12) in Keeper snapshots [#24499](https://github.com/ClickHouse/ClickHouse/pull/24499) ([alesapin](https://github.com/alesapin)). +* [backport] Find only opened PRs by label [#24501](https://github.com/ClickHouse/ClickHouse/pull/24501) ([Ivan](https://github.com/abyss7)). +* shellcheck fix [#24512](https://github.com/ClickHouse/ClickHouse/pull/24512) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not peerdir enabled-by-default libcxxabi-parts [#24518](https://github.com/ClickHouse/ClickHouse/pull/24518) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Merging [#22762](https://github.com/ClickHouse/ClickHouse/issues/22762) [#24525](https://github.com/ClickHouse/ClickHouse/pull/24525) ([Vladimir C](https://github.com/vdimir)). +* Fix Arcadia [#24527](https://github.com/ClickHouse/ClickHouse/pull/24527) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make `get/check/read/list` requests always read in Keeper [#24533](https://github.com/ClickHouse/ClickHouse/pull/24533) ([alesapin](https://github.com/alesapin)). +* calculate perf test precision thresholds from historical data [#24534](https://github.com/ClickHouse/ClickHouse/pull/24534) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update jit_example [#24550](https://github.com/ClickHouse/ClickHouse/pull/24550) ([Maksim Kita](https://github.com/kitaisreal)). +* Update test_multiple_disks [#24560](https://github.com/ClickHouse/ClickHouse/pull/24560) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test [#24561](https://github.com/ClickHouse/ClickHouse/pull/24561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arcadia [#24563](https://github.com/ClickHouse/ClickHouse/pull/24563) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix avro [#24569](https://github.com/ClickHouse/ClickHouse/pull/24569) ([Ilya Yatsishin](https://github.com/qoega)). +* Refactor MergeTreeDataSelectExecutor [#24574](https://github.com/ClickHouse/ClickHouse/pull/24574) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* allow inheriting from a named window in window definition [#24576](https://github.com/ClickHouse/ClickHouse/pull/24576) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* do not auto-apply -OrNull combinator to pure window functions [#24579](https://github.com/ClickHouse/ClickHouse/pull/24579) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix ORDER BY after window fuctions over Distributed [#24580](https://github.com/ClickHouse/ClickHouse/pull/24580) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ast fuzzer: determine server death more robustly [#24584](https://github.com/ClickHouse/ClickHouse/pull/24584) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable tests, utils and examples in builds with clang-tidy [#24587](https://github.com/ClickHouse/ClickHouse/pull/24587) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* more fuzzer fixes [#24690](https://github.com/ClickHouse/ClickHouse/pull/24690) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flaky integration tests [#24691](https://github.com/ClickHouse/ClickHouse/pull/24691) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix alter table drop projection if exists [#24692](https://github.com/ClickHouse/ClickHouse/pull/24692) ([Amos Bird](https://github.com/amosbird)). +* Block memory tracker earlier in `tryLogCurrentException` [#24722](https://github.com/ClickHouse/ClickHouse/pull/24722) ([alesapin](https://github.com/alesapin)). +* Minor fixes in AggregateFunctionSegmentLengthSumData [#24729](https://github.com/ClickHouse/ClickHouse/pull/24729) ([Vladimir C](https://github.com/vdimir)). +* remove mutable references to Context from IFunction interface [#24732](https://github.com/ClickHouse/ClickHouse/pull/24732) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* MemoryTracker enable throw logical error [#24733](https://github.com/ClickHouse/ClickHouse/pull/24733) ([Maksim Kita](https://github.com/kitaisreal)). +* support expressions in window frame [#24734](https://github.com/ClickHouse/ClickHouse/pull/24734) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Updated libunwind [#24735](https://github.com/ClickHouse/ClickHouse/pull/24735) ([Maksim Kita](https://github.com/kitaisreal)). +* ExecuteScalarSubqueriesVisitor fix error code style check [#24736](https://github.com/ClickHouse/ClickHouse/pull/24736) ([Maksim Kita](https://github.com/kitaisreal)). +* Process config w/o extensions as XML format [#24763](https://github.com/ClickHouse/ClickHouse/pull/24763) ([Azat Khuzhin](https://github.com/azat)). +* Update NuRaft [#24775](https://github.com/ClickHouse/ClickHouse/pull/24775) ([alesapin](https://github.com/alesapin)). +* Fix empty part set with force_use_projection = 1 [#24782](https://github.com/ClickHouse/ClickHouse/pull/24782) ([Amos Bird](https://github.com/amosbird)). +* Better exception for invalid projection creation [#24785](https://github.com/ClickHouse/ClickHouse/pull/24785) ([Amos Bird](https://github.com/amosbird)). +* Improve "Cannot schedule a task" error message [#24786](https://github.com/ClickHouse/ClickHouse/pull/24786) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00953_zookeeper_suetin_deduplication_bug [#24801](https://github.com/ClickHouse/ClickHouse/pull/24801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't capture temporary references in ZooKeeper client callbacks. [#24803](https://github.com/ClickHouse/ClickHouse/pull/24803) ([alesapin](https://github.com/alesapin)). +* Small improvement for StorageMaterializedView::getActionLock(...) [#24806](https://github.com/ClickHouse/ClickHouse/pull/24806) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Make ContextPtr const by default. [#24808](https://github.com/ClickHouse/ClickHouse/pull/24808) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix typo. Remove the "allow_experimental_bigint_types" setting. [#24812](https://github.com/ClickHouse/ClickHouse/pull/24812) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ExpressionJIT remove unncecessary logging [#24813](https://github.com/ClickHouse/ClickHouse/pull/24813) ([Maksim Kita](https://github.com/kitaisreal)). +* ExpressionJIT simplify loop [#24814](https://github.com/ClickHouse/ClickHouse/pull/24814) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed clang tidy [#24821](https://github.com/ClickHouse/ClickHouse/pull/24821) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#23260](https://github.com/ClickHouse/ClickHouse/issues/23260) [#24822](https://github.com/ClickHouse/ClickHouse/pull/24822) ([Anton Popov](https://github.com/CurtizJ)). +* Simplify code around TraceCollector, alternative to [#24829](https://github.com/ClickHouse/ClickHouse/issues/24829) [#24833](https://github.com/ClickHouse/ClickHouse/pull/24833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Some improvements in stress test [#24835](https://github.com/ClickHouse/ClickHouse/pull/24835) ([alesapin](https://github.com/alesapin)). +* Rename ContextConstPtr to ContextPtr. [#24855](https://github.com/ClickHouse/ClickHouse/pull/24855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reverted libunwind from upstream [#24860](https://github.com/ClickHouse/ClickHouse/pull/24860) ([Maksim Kita](https://github.com/kitaisreal)). +* Testflows Extended Precision Data Type testing - Adding snapshots for output comparison. [#24861](https://github.com/ClickHouse/ClickHouse/pull/24861) ([MyroTk](https://github.com/MyroTk)). +* clickhouse-client: echo hint improvements [#24863](https://github.com/ClickHouse/ClickHouse/pull/24863) ([Azat Khuzhin](https://github.com/azat)). +* docs: update requests (to fix conflicts with urllib3) [#24865](https://github.com/ClickHouse/ClickHouse/pull/24865) ([Azat Khuzhin](https://github.com/azat)). +* Trying to resurrect woboq [#24875](https://github.com/ClickHouse/ClickHouse/pull/24875) ([alesapin](https://github.com/alesapin)). +* Get rid of std::stringstream in Suggest [#24882](https://github.com/ClickHouse/ClickHouse/pull/24882) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Suppress RocksDB error [#24886](https://github.com/ClickHouse/ClickHouse/pull/24886) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove some outdated integration tests [#24911](https://github.com/ClickHouse/ClickHouse/pull/24911) ([alesapin](https://github.com/alesapin)). +* Mute test_memory_consumption before [#24784](https://github.com/ClickHouse/ClickHouse/issues/24784) [#24919](https://github.com/ClickHouse/ClickHouse/pull/24919) ([Ilya Yatsishin](https://github.com/qoega)). +* Update something in KeyCondition [#24920](https://github.com/ClickHouse/ClickHouse/pull/24920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Relax hung check [#24929](https://github.com/ClickHouse/ClickHouse/pull/24929) ([alesapin](https://github.com/alesapin)). +* some perf test script improvements [#24938](https://github.com/ClickHouse/ClickHouse/pull/24938) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* RFC: clickhouse-test: do not substitude db name in stderr by default [#24939](https://github.com/ClickHouse/ClickHouse/pull/24939) ([Azat Khuzhin](https://github.com/azat)). +* Add test issue [#23430](https://github.com/ClickHouse/ClickHouse/issues/23430) [#24941](https://github.com/ClickHouse/ClickHouse/pull/24941) ([filimonov](https://github.com/filimonov)). +* Followup fixes for integration tests [#24954](https://github.com/ClickHouse/ClickHouse/pull/24954) ([alesapin](https://github.com/alesapin)). +* Fix bad error message in docker entrypoint [#24955](https://github.com/ClickHouse/ClickHouse/pull/24955) ([filimonov](https://github.com/filimonov)). +* Fix endless wait in replica clone [#24957](https://github.com/ClickHouse/ClickHouse/pull/24957) ([alesapin](https://github.com/alesapin)). +* Remove subprocess_call from cluster.py [#24959](https://github.com/ClickHouse/ClickHouse/pull/24959) ([Ilya Yatsishin](https://github.com/qoega)). +* Delete support for waiting on queue- entries, is this dead code? [#24960](https://github.com/ClickHouse/ClickHouse/pull/24960) ([nvartolomei](https://github.com/nvartolomei)). +* Fix the test after [#20393](https://github.com/ClickHouse/ClickHouse/issues/20393) [#24967](https://github.com/ClickHouse/ClickHouse/pull/24967) ([filimonov](https://github.com/filimonov)). +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix typo in usage section in s3 documentation [#24970](https://github.com/ClickHouse/ClickHouse/pull/24970) ([presto53](https://github.com/presto53)). +* Cleanup iptables and containers on session start [#24973](https://github.com/ClickHouse/ClickHouse/pull/24973) ([Ilya Yatsishin](https://github.com/qoega)). +* Part movement between shards ACL [#24979](https://github.com/ClickHouse/ClickHouse/pull/24979) ([nvartolomei](https://github.com/nvartolomei)). +* Cleanup changelog script [#24987](https://github.com/ClickHouse/ClickHouse/pull/24987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.6 [#24989](https://github.com/ClickHouse/ClickHouse/pull/24989) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* clickhouse-test: fix log_comment for .sql tests [#24999](https://github.com/ClickHouse/ClickHouse/pull/24999) ([Azat Khuzhin](https://github.com/azat)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). +* Compression codecs refactoring [#25002](https://github.com/ClickHouse/ClickHouse/pull/25002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe Minio starts for too long in tests [#25007](https://github.com/ClickHouse/ClickHouse/pull/25007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad test [#25012](https://github.com/ClickHouse/ClickHouse/pull/25012) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disks fix ya.make [#25031](https://github.com/ClickHouse/ClickHouse/pull/25031) ([Maksim Kita](https://github.com/kitaisreal)). +* tests: disable suggestions for expect tests that does not requires it [#25033](https://github.com/ClickHouse/ClickHouse/pull/25033) ([Azat Khuzhin](https://github.com/azat)). +* fix jemalloc build on OS_DARWIN [#25034](https://github.com/ClickHouse/ClickHouse/pull/25034) ([sdk2](https://github.com/sdk2)). +* Update waitForTableReplicaToProcessLogEntry comments [#25037](https://github.com/ClickHouse/ClickHouse/pull/25037) ([nvartolomei](https://github.com/nvartolomei)). +* Remove database before model name in docs [#25038](https://github.com/ClickHouse/ClickHouse/pull/25038) ([Kruglov Pavel](https://github.com/Avogar)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). +* test for attach partition from [#25060](https://github.com/ClickHouse/ClickHouse/pull/25060) ([Denny Crane](https://github.com/den-crane)). +* AggregateFunctionAnyHeavyData use fixed size type [#25066](https://github.com/ClickHouse/ClickHouse/pull/25066) ([Maksim Kita](https://github.com/kitaisreal)). +* Update Dockerfile [#25078](https://github.com/ClickHouse/ClickHouse/pull/25078) ([Ivan](https://github.com/abyss7)). +* Trying to debug fetches bandwith test failures [#25079](https://github.com/ClickHouse/ClickHouse/pull/25079) ([alesapin](https://github.com/alesapin)). +* Remove strange timeout in test [#25084](https://github.com/ClickHouse/ClickHouse/pull/25084) ([alesapin](https://github.com/alesapin)). +* Remove copypaste from StorageReplicatedMergeTree [#25087](https://github.com/ClickHouse/ClickHouse/pull/25087) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Less timeouts in integration tests [#25111](https://github.com/ClickHouse/ClickHouse/pull/25111) ([alesapin](https://github.com/alesapin)). +* Better odbc integration test [#25113](https://github.com/ClickHouse/ClickHouse/pull/25113) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flaky check report in CI runner [#25126](https://github.com/ClickHouse/ClickHouse/pull/25126) ([alesapin](https://github.com/alesapin)). +* Better hdfs tests [#25128](https://github.com/ClickHouse/ClickHouse/pull/25128) ([Ilya Yatsishin](https://github.com/qoega)). +* Use zstd/include in ya.make [#25145](https://github.com/ClickHouse/ClickHouse/pull/25145) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Adds a better way to include binary resources [#25146](https://github.com/ClickHouse/ClickHouse/pull/25146) ([bnaecker](https://github.com/bnaecker)). +* check for row_policy defined using user() function [#25147](https://github.com/ClickHouse/ClickHouse/pull/25147) ([Denny Crane](https://github.com/den-crane)). +* Update Roaring Bitmaps just in case [#25151](https://github.com/ClickHouse/ClickHouse/pull/25151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move code to more appropriate place [#25152](https://github.com/ClickHouse/ClickHouse/pull/25152) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionary sources add update lag option [#25161](https://github.com/ClickHouse/ClickHouse/pull/25161) ([Maksim Kita](https://github.com/kitaisreal)). +* RewriteFunctionToSubcolumnVisitor add into ya.make [#25162](https://github.com/ClickHouse/ClickHouse/pull/25162) ([Maksim Kita](https://github.com/kitaisreal)). +* Enable back the ANTLR in .sql [#25170](https://github.com/ClickHouse/ClickHouse/pull/25170) ([Ivan](https://github.com/abyss7)). +* Added ExecutablePool documentation [#25196](https://github.com/ClickHouse/ClickHouse/pull/25196) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not optimize query plan for mutations. [#25197](https://github.com/ClickHouse/ClickHouse/pull/25197) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Dictionary added update field documentation [#25198](https://github.com/ClickHouse/ClickHouse/pull/25198) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries attribute support default nullable type [#25203](https://github.com/ClickHouse/ClickHouse/pull/25203) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix tests [#25204](https://github.com/ClickHouse/ClickHouse/pull/25204) ([Ivan](https://github.com/abyss7)). +* Try run tests with Replicated database in parallel [#25210](https://github.com/ClickHouse/ClickHouse/pull/25210) ([Alexander Tokmakov](https://github.com/tavplubix)). +* More integration tests improvements [#25212](https://github.com/ClickHouse/ClickHouse/pull/25212) ([Ilya Yatsishin](https://github.com/qoega)). +* Add function toJSONString to ya.make [#25246](https://github.com/ClickHouse/ClickHouse/pull/25246) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor change [#25247](https://github.com/ClickHouse/ClickHouse/pull/25247) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add missed #include [#25248](https://github.com/ClickHouse/ClickHouse/pull/25248) ([Matwey V. Kornilov](https://github.com/matwey)). +* Add test for [#4113](https://github.com/ClickHouse/ClickHouse/issues/4113) [#25249](https://github.com/ClickHouse/ClickHouse/pull/25249) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#11535](https://github.com/ClickHouse/ClickHouse/issues/11535) [#25250](https://github.com/ClickHouse/ClickHouse/pull/25250) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9932](https://github.com/ClickHouse/ClickHouse/issues/9932) [#25253](https://github.com/ClickHouse/ClickHouse/pull/25253) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Copy-paste some code [#25264](https://github.com/ClickHouse/ClickHouse/pull/25264) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor change [#25265](https://github.com/ClickHouse/ClickHouse/pull/25265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315) [#25266](https://github.com/ClickHouse/ClickHouse/pull/25266) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add missed #include [#25267](https://github.com/ClickHouse/ClickHouse/pull/25267) ([Matwey V. Kornilov](https://github.com/matwey)). +* test for `PARTITION BY 0 * id` [#25274](https://github.com/ClickHouse/ClickHouse/pull/25274) ([Denny Crane](https://github.com/den-crane)). +* Enable TestFlows LDAP tests [#25278](https://github.com/ClickHouse/ClickHouse/pull/25278) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add a test for [#17964](https://github.com/ClickHouse/ClickHouse/issues/17964) [#25285](https://github.com/ClickHouse/ClickHouse/pull/25285) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#17367](https://github.com/ClickHouse/ClickHouse/issues/17367) [#25286](https://github.com/ClickHouse/ClickHouse/pull/25286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some code from KeyCondition. [#25295](https://github.com/ClickHouse/ClickHouse/pull/25295) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CMake dictionaries disable debug info fix [#25301](https://github.com/ClickHouse/ClickHouse/pull/25301) ([Maksim Kita](https://github.com/kitaisreal)). +* Merge ext into common [#25303](https://github.com/ClickHouse/ClickHouse/pull/25303) ([Maksim Kita](https://github.com/kitaisreal)). +* test for null array orc load [#25304](https://github.com/ClickHouse/ClickHouse/pull/25304) ([Denny Crane](https://github.com/den-crane)). +* Improve Replicated database tests [#25305](https://github.com/ClickHouse/ClickHouse/pull/25305) ([Alexander Tokmakov](https://github.com/tavplubix)). +* SimpleCache key constructor improvement [#25307](https://github.com/ClickHouse/ClickHouse/pull/25307) ([Maksim Kita](https://github.com/kitaisreal)). +* Catch ErrnoException during parts cleaning [#25309](https://github.com/ClickHouse/ClickHouse/pull/25309) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test 01520_client_print_query_id and others. [#25311](https://github.com/ClickHouse/ClickHouse/pull/25311) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileTiming [#25314](https://github.com/ClickHouse/ClickHouse/pull/25314) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update test_ttl_replicated [#25317](https://github.com/ClickHouse/ClickHouse/pull/25317) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `00601_kill_running_query` [#25318](https://github.com/ClickHouse/ClickHouse/pull/25318) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use signal 15 in restart_with_latest_version [#25323](https://github.com/ClickHouse/ClickHouse/pull/25323) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Setting min_count_to_compile_expression fix [#25332](https://github.com/ClickHouse/ClickHouse/pull/25332) ([Maksim Kita](https://github.com/kitaisreal)). +* Function formatDateTime fix code comments [#25334](https://github.com/ClickHouse/ClickHouse/pull/25334) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve Replicated database tests 2 [#25373](https://github.com/ClickHouse/ClickHouse/pull/25373) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a test for [#23163](https://github.com/ClickHouse/ClickHouse/issues/23163) [#25390](https://github.com/ClickHouse/ClickHouse/pull/25390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rename & reimport murmurhash sources from smhasher repo [#25400](https://github.com/ClickHouse/ClickHouse/pull/25400) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Add test from issue [#20624](https://github.com/ClickHouse/ClickHouse/issues/20624) [#25409](https://github.com/ClickHouse/ClickHouse/pull/25409) ([Vladimir C](https://github.com/vdimir)). +* Turn off WITH_COVERAGE in build with clang-tidy [#25417](https://github.com/ClickHouse/ClickHouse/pull/25417) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix container-overflow in replxx during incremental search (Ctrl-R) [#25427](https://github.com/ClickHouse/ClickHouse/pull/25427) ([Azat Khuzhin](https://github.com/azat)). +* DatabaseMySQL rename [#25430](https://github.com/ClickHouse/ClickHouse/pull/25430) ([Maksim Kita](https://github.com/kitaisreal)). +* Support REPLACE DICTIONARY, CREATE OR REPLACE DICTIONARY queries [#25444](https://github.com/ClickHouse/ClickHouse/pull/25444) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8417](https://github.com/ClickHouse/ClickHouse/issues/8417) [#25446](https://github.com/ClickHouse/ClickHouse/pull/25446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove assumeMutable from removeColumnNullability [#25454](https://github.com/ClickHouse/ClickHouse/pull/25454) ([Vladimir C](https://github.com/vdimir)). +* DataTypeLowCardinality support DataTypeInterval tests [#25458](https://github.com/ClickHouse/ClickHouse/pull/25458) ([Maksim Kita](https://github.com/kitaisreal)). +* Add run-id option to integration tests [#25459](https://github.com/ClickHouse/ClickHouse/pull/25459) ([alesapin](https://github.com/alesapin)). +* Stable NOT chain formatting [#25494](https://github.com/ClickHouse/ClickHouse/pull/25494) ([Azat Khuzhin](https://github.com/azat)). +* Fix alternative stack for SIGSEGV handling [#25509](https://github.com/ClickHouse/ClickHouse/pull/25509) ([Azat Khuzhin](https://github.com/azat)). +* Catch "Maximum parse depth" error in fuzzer [#25510](https://github.com/ClickHouse/ClickHouse/pull/25510) ([Azat Khuzhin](https://github.com/azat)). +* Fix NOT parsing [#25520](https://github.com/ClickHouse/ClickHouse/pull/25520) ([Azat Khuzhin](https://github.com/azat)). +* Catch TOO_DEEP_RECURSION in fuzzer for formatted query too [#25521](https://github.com/ClickHouse/ClickHouse/pull/25521) ([Azat Khuzhin](https://github.com/azat)). +* Fix some bugs in integration tests [#25525](https://github.com/ClickHouse/ClickHouse/pull/25525) ([alesapin](https://github.com/alesapin)). +* Fix query progress [#25545](https://github.com/ClickHouse/ClickHouse/pull/25545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add test for progress bar [#25551](https://github.com/ClickHouse/ClickHouse/pull/25551) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix some tests [#25564](https://github.com/ClickHouse/ClickHouse/pull/25564) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test 01249_flush_interactive.sh [#25565](https://github.com/ClickHouse/ClickHouse/pull/25565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* METR-41529 [#25568](https://github.com/ClickHouse/ClickHouse/pull/25568) ([egatov](https://github.com/egatov)). +* clickhouse-client: fix NULL dereference for --param w/o value [#25579](https://github.com/ClickHouse/ClickHouse/pull/25579) ([Azat Khuzhin](https://github.com/azat)). +* Remove only symlinks during force_restore_data of Atomic engine [#25582](https://github.com/ClickHouse/ClickHouse/pull/25582) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: use basename (instead of full path) for log_comment [#25583](https://github.com/ClickHouse/ClickHouse/pull/25583) ([Azat Khuzhin](https://github.com/azat)). +* Fix typo in hardware failure error message [#25584](https://github.com/ClickHouse/ClickHouse/pull/25584) ([Stas Kelvich](https://github.com/kelvich)). +* Small change in Roaring Bitmaps [#25604](https://github.com/ClickHouse/ClickHouse/pull/25604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ANTLR parser and enable it back in CI [#25638](https://github.com/ClickHouse/ClickHouse/pull/25638) ([Ivan](https://github.com/abyss7)). +* Fix alternative stack (MINSIGSTKSZ) and stack size check under osx [#25654](https://github.com/ClickHouse/ClickHouse/pull/25654) ([Azat Khuzhin](https://github.com/azat)). +* Enable MurmurHash in ArcadiaBuild [#25666](https://github.com/ClickHouse/ClickHouse/pull/25666) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Add working test [#25720](https://github.com/ClickHouse/ClickHouse/pull/25720) ([alesapin](https://github.com/alesapin)). +* Compile expressions updated documentation [#25725](https://github.com/ClickHouse/ClickHouse/pull/25725) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix some more missed includes [#25732](https://github.com/ClickHouse/ClickHouse/pull/25732) ([Matwey V. Kornilov](https://github.com/matwey)). +* Fix native macOS (Xcode) builds [#25736](https://github.com/ClickHouse/ClickHouse/pull/25736) ([Denis Glazachev](https://github.com/traceon)). +* Fix arcadia [#25738](https://github.com/ClickHouse/ClickHouse/pull/25738) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Change error code in LIVE VIEW [#25739](https://github.com/ClickHouse/ClickHouse/pull/25739) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### Testing Improvement * * Add join related options to stress tests. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v21.7.10.4-stable.md b/docs/changelogs/v21.7.10.4-stable.md index 9056da8ac89..963ac6d6ac7 100644 --- a/docs/changelogs/v21.7.10.4-stable.md +++ b/docs/changelogs/v21.7.10.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.10.4-stable FIXME as compared to v21.7.9.7-stable #### Improvement @@ -17,3 +24,11 @@ * Backported in [#28947](https://github.com/ClickHouse/ClickHouse/issues/28947): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28931](https://github.com/ClickHouse/ClickHouse/issues/28931): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.7.11.3-stable.md b/docs/changelogs/v21.7.11.3-stable.md index 66672204713..e130550fb9c 100644 --- a/docs/changelogs/v21.7.11.3-stable.md +++ b/docs/changelogs/v21.7.11.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -5,3 +12,9 @@ * Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.2.7-stable.md b/docs/changelogs/v21.7.2.7-stable.md index a7af0fed667..5c772eab664 100644 --- a/docs/changelogs/v21.7.2.7-stable.md +++ b/docs/changelogs/v21.7.2.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.2.7-stable FIXME as compared to v21.7.1.7283-prestable #### Improvement @@ -15,3 +22,8 @@ * Backported in [#26010](https://github.com/ClickHouse/ClickHouse/issues/26010): Fix formatting of type `Map` with integer keys to `JSON`. [#25982](https://github.com/ClickHouse/ClickHouse/pull/25982) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#26097](https://github.com/ClickHouse/ClickHouse/issues/26097): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). +* Proper fix of serialization of type Map to JSON [#26048](https://github.com/ClickHouse/ClickHouse/pull/26048) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.7.3.14-stable.md b/docs/changelogs/v21.7.3.14-stable.md index d24b7bcbf39..0ad15f765a1 100644 --- a/docs/changelogs/v21.7.3.14-stable.md +++ b/docs/changelogs/v21.7.3.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.3.14-stable FIXME as compared to v21.7.2.7-stable #### Bug Fix @@ -9,3 +16,7 @@ * Backported in [#26229](https://github.com/ClickHouse/ClickHouse/issues/26229): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix throwing exception when iterate over non existing remote directory. [#26296](https://github.com/ClickHouse/ClickHouse/pull/26296) ([ianton-ru](https://github.com/ianton-ru)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.4.18-stable.md b/docs/changelogs/v21.7.4.18-stable.md index 7bc08e2a0e3..34d457015cb 100644 --- a/docs/changelogs/v21.7.4.18-stable.md +++ b/docs/changelogs/v21.7.4.18-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.4.18-stable FIXME as compared to v21.7.3.14-stable #### Bug Fix diff --git a/docs/changelogs/v21.7.5.29-stable.md b/docs/changelogs/v21.7.5.29-stable.md index 3f24e3eded9..6b01d912e17 100644 --- a/docs/changelogs/v21.7.5.29-stable.md +++ b/docs/changelogs/v21.7.5.29-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.5.29-stable FIXME as compared to v21.7.4.18-stable #### Performance Improvement @@ -14,3 +21,10 @@ * Backported in [#26772](https://github.com/ClickHouse/ClickHouse/issues/26772): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#26907](https://github.com/ClickHouse/ClickHouse/issues/26907): Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.6.39-stable.md b/docs/changelogs/v21.7.6.39-stable.md index a7913aca193..03afc95515e 100644 --- a/docs/changelogs/v21.7.6.39-stable.md +++ b/docs/changelogs/v21.7.6.39-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.6.39-stable FIXME as compared to v21.7.5.29-stable #### Bug Fix @@ -12,3 +19,7 @@ * Backported in [#27156](https://github.com/ClickHouse/ClickHouse/issues/27156): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#27261](https://github.com/ClickHouse/ClickHouse/issues/27261): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.7.47-stable.md b/docs/changelogs/v21.7.7.47-stable.md index f81abca2600..3c7bf09433e 100644 --- a/docs/changelogs/v21.7.7.47-stable.md +++ b/docs/changelogs/v21.7.7.47-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.7.47-stable FIXME as compared to v21.7.6.39-stable #### Bug Fix @@ -6,3 +13,12 @@ * Backported in [#27418](https://github.com/ClickHouse/ClickHouse/issues/27418): Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#27416](https://github.com/ClickHouse/ClickHouse/issues/27416): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix parallel execution of integration tests [#25986](https://github.com/ClickHouse/ClickHouse/pull/25986) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.8.58-stable.md b/docs/changelogs/v21.7.8.58-stable.md index aea1ae083f0..4f99e5439a4 100644 --- a/docs/changelogs/v21.7.8.58-stable.md +++ b/docs/changelogs/v21.7.8.58-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.8.58-stable FIXME as compared to v21.7.7.47-stable #### Bug Fix @@ -10,3 +17,8 @@ * Backported in [#27697](https://github.com/ClickHouse/ClickHouse/issues/27697): Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#27748](https://github.com/ClickHouse/ClickHouse/issues/27748): Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md index 0d1a2a521e7..1d859b08da5 100644 --- a/docs/changelogs/v21.7.9.7-stable.md +++ b/docs/changelogs/v21.7.9.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.9.7-stable FIXME as compared to v21.7.8.58-stable #### Improvement @@ -25,3 +32,15 @@ * Backported in [#28288](https://github.com/ClickHouse/ClickHouse/issues/28288): Fix reading of custom TLD w/o new line at EOF. [#28213](https://github.com/ClickHouse/ClickHouse/pull/28213) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#28295](https://github.com/ClickHouse/ClickHouse/issues/28295): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.1.7409-prestable.md b/docs/changelogs/v21.8.1.7409-prestable.md index e703d227603..96f13713984 100644 --- a/docs/changelogs/v21.8.1.7409-prestable.md +++ b/docs/changelogs/v21.8.1.7409-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.1.7409-prestable FIXME as compared to v21.7.1.7283-prestable #### Backward Incompatible Change @@ -86,3 +93,62 @@ * NO CL ENTRY: '[ImgBot] Optimize images'. [#26054](https://github.com/ClickHouse/ClickHouse/pull/26054) ([imgbot[bot]](https://github.com/apps/imgbot)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix hang and incorrect exit code returned from clickhouse-test [#25537](https://github.com/ClickHouse/ClickHouse/pull/25537) ([nvartolomei](https://github.com/nvartolomei)). +* Remove PrewhereDAGInfo. [#25719](https://github.com/ClickHouse/ClickHouse/pull/25719) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix 01641_memory_tracking_insert_optimize [#25731](https://github.com/ClickHouse/ClickHouse/pull/25731) ([Azat Khuzhin](https://github.com/azat)). +* Separate log files for separate runs in stress test [#25741](https://github.com/ClickHouse/ClickHouse/pull/25741) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix slow performance test [#25742](https://github.com/ClickHouse/ClickHouse/pull/25742) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DatabaseAtomic EXCHANGE DICTIONARIES fix test [#25753](https://github.com/ClickHouse/ClickHouse/pull/25753) ([Maksim Kita](https://github.com/kitaisreal)). +* Try fix flacky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#13993](https://github.com/ClickHouse/ClickHouse/issues/13993) [#25758](https://github.com/ClickHouse/ClickHouse/pull/25758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set follow-fork-mode child for gdb in stress/fasttest/fuzzer [#25769](https://github.com/ClickHouse/ClickHouse/pull/25769) ([Azat Khuzhin](https://github.com/azat)). +* Ignore TOO_DEEP_RECURSION server exception during fuzzing [#25770](https://github.com/ClickHouse/ClickHouse/pull/25770) ([Azat Khuzhin](https://github.com/azat)). +* Add comments for VERSION_REVISION vs DBMS_TCP_PROTOCOL_VERSION [#25771](https://github.com/ClickHouse/ClickHouse/pull/25771) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test and wrong message [#25772](https://github.com/ClickHouse/ClickHouse/pull/25772) ([alesapin](https://github.com/alesapin)). +* MaterializeMySQL: Improved column comments support [#25781](https://github.com/ClickHouse/ClickHouse/pull/25781) ([Storozhuk Kostiantyn](https://github.com/sand6255)). +* Fix ANTRL merge_prewhere_table test [#25782](https://github.com/ClickHouse/ClickHouse/pull/25782) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove dead code from VirtualColumnUtils.cpp [#25787](https://github.com/ClickHouse/ClickHouse/pull/25787) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix some typos in Storage classes [#25796](https://github.com/ClickHouse/ClickHouse/pull/25796) ([Raúl Marín](https://github.com/Algunenano)). +* Fix DateLUT on Darwin [#25803](https://github.com/ClickHouse/ClickHouse/pull/25803) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better test_version_update_after_mutation [#25810](https://github.com/ClickHouse/ClickHouse/pull/25810) ([alesapin](https://github.com/alesapin)). +* Collect stderr.log and stdout.log in all integration tests by default. [#25816](https://github.com/ClickHouse/ClickHouse/pull/25816) ([Vitaly Baranov](https://github.com/vitlibar)). +* METR-41529 [#25819](https://github.com/ClickHouse/ClickHouse/pull/25819) ([egatov](https://github.com/egatov)). +* tests/integration: use iptables --wait [#25823](https://github.com/ClickHouse/ClickHouse/pull/25823) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for [#25611](https://github.com/ClickHouse/ClickHouse/issues/25611) [#25831](https://github.com/ClickHouse/ClickHouse/pull/25831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix odbc test [#25834](https://github.com/ClickHouse/ClickHouse/pull/25834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix performance tests after converting ProfileEvents to Map type [#25837](https://github.com/ClickHouse/ClickHouse/pull/25837) ([Azat Khuzhin](https://github.com/azat)). +* Correct messages in integration tests. [#25861](https://github.com/ClickHouse/ClickHouse/pull/25861) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix typo in the test 00900_orc_arrow_parquet_maps [#25874](https://github.com/ClickHouse/ClickHouse/pull/25874) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve logging in integration tests. [#25899](https://github.com/ClickHouse/ClickHouse/pull/25899) ([Vitaly Baranov](https://github.com/vitlibar)). +* kerberized HDFS test fix if run in parallel [#25908](https://github.com/ClickHouse/ClickHouse/pull/25908) ([Ilya Golshtein](https://github.com/ilejn)). +* fix special build on clang 11 [#25912](https://github.com/ClickHouse/ClickHouse/pull/25912) ([flynn](https://github.com/ucasfl)). +* Remove obsolete code from init script [#25920](https://github.com/ClickHouse/ClickHouse/pull/25920) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionInitializeAggregation build fix [#25922](https://github.com/ClickHouse/ClickHouse/pull/25922) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix clang tidy build check [#25939](https://github.com/ClickHouse/ClickHouse/pull/25939) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build subquery in replaceJoinedTable without parsing [#25941](https://github.com/ClickHouse/ClickHouse/pull/25941) ([Vladimir C](https://github.com/vdimir)). +* Remove experimental ANTLR parser [#25942](https://github.com/ClickHouse/ClickHouse/pull/25942) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow-up for [#20470](https://github.com/ClickHouse/ClickHouse/issues/20470) [#25975](https://github.com/ClickHouse/ClickHouse/pull/25975) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Correct test [#25984](https://github.com/ClickHouse/ClickHouse/pull/25984) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix parallel execution of integration tests [#25986](https://github.com/ClickHouse/ClickHouse/pull/25986) ([Vitaly Baranov](https://github.com/vitlibar)). +* Compile aggregate functions perf tests fix [#25989](https://github.com/ClickHouse/ClickHouse/pull/25989) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix alter of settings in `MergeTree` tables [#25995](https://github.com/ClickHouse/ClickHouse/pull/25995) ([Anton Popov](https://github.com/CurtizJ)). +* Fix arcadia [#26002](https://github.com/ClickHouse/ClickHouse/pull/26002) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove old code [#26014](https://github.com/ClickHouse/ClickHouse/pull/26014) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionsLogical const result for non const arguments fix [#26018](https://github.com/ClickHouse/ClickHouse/pull/26018) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionSQLJSON ContextPtr build fix [#26022](https://github.com/ClickHouse/ClickHouse/pull/26022) ([Maksim Kita](https://github.com/kitaisreal)). +* Replace print() with logging.debug() in integration tests. [#26023](https://github.com/ClickHouse/ClickHouse/pull/26023) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try to fix some flaky tests [#26032](https://github.com/ClickHouse/ClickHouse/pull/26032) ([Anton Popov](https://github.com/CurtizJ)). +* Fix for ZK watch metric drift in rare conditions [#26034](https://github.com/ClickHouse/ClickHouse/pull/26034) ([nvartolomei](https://github.com/nvartolomei)). +* AsynchronousMetrics: Don't assume temperature is always positive [#26045](https://github.com/ClickHouse/ClickHouse/pull/26045) ([Raúl Marín](https://github.com/Algunenano)). +* Proper fix of serialization of type Map to JSON [#26048](https://github.com/ClickHouse/ClickHouse/pull/26048) ([Anton Popov](https://github.com/CurtizJ)). +* ClickHouse dictionary source secure setting added documentation [#26055](https://github.com/ClickHouse/ClickHouse/pull/26055) ([Maksim Kita](https://github.com/kitaisreal)). +* Add changelog for 21.7 [#26057](https://github.com/ClickHouse/ClickHouse/pull/26057) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix 01791_dist_INSERT_block_structure_mismatch flakiness [#26058](https://github.com/ClickHouse/ClickHouse/pull/26058) ([Azat Khuzhin](https://github.com/azat)). +* Fix logical error with signed and unsigned offset in WindowFrame::checkValid [#26072](https://github.com/ClickHouse/ClickHouse/pull/26072) ([Vladimir C](https://github.com/vdimir)). +* Remove unused code [#26077](https://github.com/ClickHouse/ClickHouse/pull/26077) ([Anton Popov](https://github.com/CurtizJ)). +* Disabling annoying copier tests [#26099](https://github.com/ClickHouse/ClickHouse/pull/26099) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make graph pipeline rendering compatible with Dagre.JS [#26114](https://github.com/ClickHouse/ClickHouse/pull/26114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.8.10.19-lts.md b/docs/changelogs/v21.8.10.19-lts.md index 92347642ffa..3e73e7892ec 100644 --- a/docs/changelogs/v21.8.10.19-lts.md +++ b/docs/changelogs/v21.8.10.19-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.10.19-lts FIXME as compared to v21.8.9.13-lts #### Improvement @@ -10,3 +17,7 @@ * Backported in [#30333](https://github.com/ClickHouse/ClickHouse/issues/30333): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). * Backported in [#30377](https://github.com/ClickHouse/ClickHouse/issues/30377): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.8.11.4-lts.md b/docs/changelogs/v21.8.11.4-lts.md index e36d75c32ea..644a9ece633 100644 --- a/docs/changelogs/v21.8.11.4-lts.md +++ b/docs/changelogs/v21.8.11.4-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.11.4-lts FIXME as compared to v21.8.10.19-lts #### New Feature @@ -38,3 +45,10 @@ * Backported in [#31132](https://github.com/ClickHouse/ClickHouse/issues/31132): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#31372](https://github.com/ClickHouse/ClickHouse/issues/31372): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix assert in table function `merge` with database regexp [#29355](https://github.com/ClickHouse/ClickHouse/pull/29355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.8.12.29-lts.md b/docs/changelogs/v21.8.12.29-lts.md index 63b34c367f3..3dea113485d 100644 --- a/docs/changelogs/v21.8.12.29-lts.md +++ b/docs/changelogs/v21.8.12.29-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.12.29-lts FIXME as compared to v21.8.11.4-lts #### Performance Improvement @@ -22,3 +29,7 @@ * Backported in [#31890](https://github.com/ClickHouse/ClickHouse/issues/31890): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31910](https://github.com/ClickHouse/ClickHouse/issues/31910): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.13.6-lts.md b/docs/changelogs/v21.8.13.6-lts.md index 06e8c366ff2..1911de83009 100644 --- a/docs/changelogs/v21.8.13.6-lts.md +++ b/docs/changelogs/v21.8.13.6-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.13.6-lts FIXME as compared to v21.8.12.29-lts #### Bug Fix @@ -27,3 +34,15 @@ * NO CL ENTRY: 'fix json error after downgrade'. [#33166](https://github.com/ClickHouse/ClickHouse/pull/33166) ([bullet1337](https://github.com/bullet1337)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 21.8 ExternalDictionariesLoader fix build [#32501](https://github.com/ClickHouse/ClickHouse/pull/32501) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 21.8 ExternalDictionariesLoader fix [#32752](https://github.com/ClickHouse/ClickHouse/pull/32752) ([Maksim Kita](https://github.com/kitaisreal)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.8.14.5-lts.md b/docs/changelogs/v21.8.14.5-lts.md index 481327a35c9..8310573f94c 100644 --- a/docs/changelogs/v21.8.14.5-lts.md +++ b/docs/changelogs/v21.8.14.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.8.15.7-lts.md b/docs/changelogs/v21.8.15.7-lts.md index 7411fbff9ae..64da1824bcf 100644 --- a/docs/changelogs/v21.8.15.7-lts.md +++ b/docs/changelogs/v21.8.15.7-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.8.2.19-prestable.md b/docs/changelogs/v21.8.2.19-prestable.md index 15726747e65..8b2bdd4e185 100644 --- a/docs/changelogs/v21.8.2.19-prestable.md +++ b/docs/changelogs/v21.8.2.19-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.2.19-prestable FIXME as compared to v21.8.1.7409-prestable #### Performance Improvement @@ -27,3 +34,10 @@ * Backported in [#26705](https://github.com/ClickHouse/ClickHouse/issues/26705): Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). * Backported in [#26771](https://github.com/ClickHouse/ClickHouse/issues/26771): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Less logging in AsynchronousMetrics [#26391](https://github.com/ClickHouse/ClickHouse/pull/26391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.8.3.44-lts.md b/docs/changelogs/v21.8.3.44-lts.md index 21fe655870a..12ecd521c15 100644 --- a/docs/changelogs/v21.8.3.44-lts.md +++ b/docs/changelogs/v21.8.3.44-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.3.44-lts FIXME as compared to v21.8.2.19-prestable #### Improvement @@ -26,3 +33,17 @@ * Backported in [#27419](https://github.com/ClickHouse/ClickHouse/issues/27419): /proc/info contains metrics like. [#27361](https://github.com/ClickHouse/ClickHouse/pull/27361) ([Mike Kot](https://github.com/myrrc)). * Backported in [#27472](https://github.com/ClickHouse/ClickHouse/issues/27472): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix assertions in Replicated database [#27033](https://github.com/ClickHouse/ClickHouse/pull/27033) ([Alexander Tokmakov](https://github.com/tavplubix)). +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.8.4.51-lts.md b/docs/changelogs/v21.8.4.51-lts.md index a8494ebb1d1..9b0dba786e5 100644 --- a/docs/changelogs/v21.8.4.51-lts.md +++ b/docs/changelogs/v21.8.4.51-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.4.51-lts FIXME as compared to v21.8.3.44-lts #### Bug Fix diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md index d78eb98b472..782c93ce9b1 100644 --- a/docs/changelogs/v21.8.5.7-lts.md +++ b/docs/changelogs/v21.8.5.7-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.5.7-lts FIXME as compared to v21.8.4.51-lts #### Improvement @@ -29,3 +36,16 @@ * Backported in [#28292](https://github.com/ClickHouse/ClickHouse/issues/28292): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28402](https://github.com/ClickHouse/ClickHouse/issues/28402): Fix intersecting parts due to new part had been replaced with an empty part. [#28310](https://github.com/ClickHouse/ClickHouse/pull/28310) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix several bugs in ZooKeeper snapshots deserialization [#26127](https://github.com/ClickHouse/ClickHouse/pull/26127) ([alesapin](https://github.com/alesapin)). +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.6.15-lts.md b/docs/changelogs/v21.8.6.15-lts.md index de38572d94a..95d349dcacb 100644 --- a/docs/changelogs/v21.8.6.15-lts.md +++ b/docs/changelogs/v21.8.6.15-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.6.15-lts FIXME as compared to v21.8.5.7-lts #### Improvement @@ -23,3 +30,12 @@ * Backported in [#28948](https://github.com/ClickHouse/ClickHouse/issues/28948): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28930](https://github.com/ClickHouse/ClickHouse/issues/28930): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix some images in release PRs [#29059](https://github.com/ClickHouse/ClickHouse/pull/29059) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.8.7.22-lts.md b/docs/changelogs/v21.8.7.22-lts.md index 92ba59f13cd..31d9c15debb 100644 --- a/docs/changelogs/v21.8.7.22-lts.md +++ b/docs/changelogs/v21.8.7.22-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,9 @@ * Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29193](https://github.com/ClickHouse/ClickHouse/issues/29193): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.8.8.29-lts.md b/docs/changelogs/v21.8.8.29-lts.md index 199be63424c..b6645362139 100644 --- a/docs/changelogs/v21.8.8.29-lts.md +++ b/docs/changelogs/v21.8.8.29-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.8.29-lts FIXME as compared to v21.8.7.22-lts #### Bug Fix diff --git a/docs/changelogs/v21.8.9.13-lts.md b/docs/changelogs/v21.8.9.13-lts.md index e4cd5f45b9b..1d9b436f302 100644 --- a/docs/changelogs/v21.8.9.13-lts.md +++ b/docs/changelogs/v21.8.9.13-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.9.13-lts FIXME as compared to v21.8.8.29-lts #### Improvement @@ -29,3 +36,9 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30187](https://github.com/ClickHouse/ClickHouse/pull/30187) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v21.9.1.8000-prestable.md b/docs/changelogs/v21.9.1.8000-prestable.md index cee357658d2..31fb71948d0 100644 --- a/docs/changelogs/v21.9.1.8000-prestable.md +++ b/docs/changelogs/v21.9.1.8000-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.1.8000-prestable FIXME as compared to v21.8.1.7409-prestable #### Backward Incompatible Change @@ -188,3 +195,209 @@ * NO CL ENTRY: 'Revert "less sys calls #2: make vdso work again"'. [#27829](https://github.com/ClickHouse/ClickHouse/pull/27829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Do not miss exceptions from the ThreadPool"'. [#27844](https://github.com/ClickHouse/ClickHouse/pull/27844) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Add comments for the implementations of the pad functions [#26147](https://github.com/ClickHouse/ClickHouse/pull/26147) ([Vitaly Baranov](https://github.com/vitlibar)). +* Change color in client for double colon [#26152](https://github.com/ClickHouse/ClickHouse/pull/26152) ([Anton Popov](https://github.com/CurtizJ)). +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Merging [#26041](https://github.com/ClickHouse/ClickHouse/issues/26041). [#26180](https://github.com/ClickHouse/ClickHouse/pull/26180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad code (default function argument) [#26213](https://github.com/ClickHouse/ClickHouse/pull/26213) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support for `pread` in `ReadBufferFromFile` [#26232](https://github.com/ClickHouse/ClickHouse/pull/26232) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ReadBufferFromS3 [#26249](https://github.com/ClickHouse/ClickHouse/pull/26249) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix output of TSV in integration tests [#26257](https://github.com/ClickHouse/ClickHouse/pull/26257) ([Anton Popov](https://github.com/CurtizJ)). +* Enum type additional support for compilation [#26258](https://github.com/ClickHouse/ClickHouse/pull/26258) ([Maksim Kita](https://github.com/kitaisreal)). +* Bump poco (now poco fork has CI via github actions) [#26262](https://github.com/ClickHouse/ClickHouse/pull/26262) ([Azat Khuzhin](https://github.com/azat)). +* Add check for sqlite database path [#26266](https://github.com/ClickHouse/ClickHouse/pull/26266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix arcadia [#26285](https://github.com/ClickHouse/ClickHouse/pull/26285) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* yandex/clickhouse-test-base Dockerfile fix (mysql connector moved) [#26294](https://github.com/ClickHouse/ClickHouse/pull/26294) ([Ilya Golshtein](https://github.com/ilejn)). +* Improve read_file_to_stringcolumn test compatibility [#26317](https://github.com/ClickHouse/ClickHouse/pull/26317) ([Raúl Marín](https://github.com/Algunenano)). +* Make socket poll() 7x faster (by replacing epoll() with poll()) [#26323](https://github.com/ClickHouse/ClickHouse/pull/26323) ([Azat Khuzhin](https://github.com/azat)). +* Modifications to an obscure Yandex TSKV format [#26326](https://github.com/ClickHouse/ClickHouse/pull/26326) ([egatov](https://github.com/egatov)). +* Fixing RBAC sample by tests in TestFlows. [#26329](https://github.com/ClickHouse/ClickHouse/pull/26329) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix error in stress test script [#26349](https://github.com/ClickHouse/ClickHouse/pull/26349) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration test about "replicated max parallel fetches". [#26362](https://github.com/ClickHouse/ClickHouse/pull/26362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Continuation of [#25774](https://github.com/ClickHouse/ClickHouse/issues/25774) [#26364](https://github.com/ClickHouse/ClickHouse/pull/26364) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enabling all TestFlows modules except LDAP after Kerberos merge. [#26366](https://github.com/ClickHouse/ClickHouse/pull/26366) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix flaky test 01293_client_interactive_vertical_multiline_long [#26367](https://github.com/ClickHouse/ClickHouse/pull/26367) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small bugfix in Block [#26373](https://github.com/ClickHouse/ClickHouse/pull/26373) ([Anton Popov](https://github.com/CurtizJ)). +* More integration tests improvements. [#26375](https://github.com/ClickHouse/ClickHouse/pull/26375) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix arcadia [#26378](https://github.com/ClickHouse/ClickHouse/pull/26378) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add separate step to read from remote. [#26381](https://github.com/ClickHouse/ClickHouse/pull/26381) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix calculating of intersection of access rights. [#26383](https://github.com/ClickHouse/ClickHouse/pull/26383) ([Vitaly Baranov](https://github.com/vitlibar)). +* Less logging in AsynchronousMetrics [#26391](https://github.com/ClickHouse/ClickHouse/pull/26391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* stress tests report improvements [#26393](https://github.com/ClickHouse/ClickHouse/pull/26393) ([Azat Khuzhin](https://github.com/azat)). +* Fix failed assertion in RocksDB in case of bad_alloc exception during batch write [#26394](https://github.com/ClickHouse/ClickHouse/pull/26394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add integrity for loaded scripts in play.html [#26409](https://github.com/ClickHouse/ClickHouse/pull/26409) ([Vladimir C](https://github.com/vdimir)). +* Relax condition in flaky test [#26425](https://github.com/ClickHouse/ClickHouse/pull/26425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Split FunctionsCoding into several files [#26426](https://github.com/ClickHouse/ClickHouse/pull/26426) ([Vladimir C](https://github.com/vdimir)). +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix "While sending batch" (on Distributed async send) [#26430](https://github.com/ClickHouse/ClickHouse/pull/26430) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes v21.9.1.7477 [#26431](https://github.com/ClickHouse/ClickHouse/pull/26431) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test_replicated_mutations (due to lack of threads in pool) [#26461](https://github.com/ClickHouse/ClickHouse/pull/26461) ([Azat Khuzhin](https://github.com/azat)). +* Fix undefined-behavior in DirectoryMonitor (for exponential back off) [#26464](https://github.com/ClickHouse/ClickHouse/pull/26464) ([Azat Khuzhin](https://github.com/azat)). +* Remove some code [#26468](https://github.com/ClickHouse/ClickHouse/pull/26468) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rewrite distributed DDL to Processors [#26469](https://github.com/ClickHouse/ClickHouse/pull/26469) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test `distributed_ddl_output_mode` [#26470](https://github.com/ClickHouse/ClickHouse/pull/26470) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). +* SELECT String from ClickHouse as Avro string - PartialMatch [#26499](https://github.com/ClickHouse/ClickHouse/pull/26499) ([Ilya Golshtein](https://github.com/ilejn)). +* Fix arcadia [#26505](https://github.com/ClickHouse/ClickHouse/pull/26505) ([Anton Popov](https://github.com/CurtizJ)). +* Fix build under AppleClang 12 [#26509](https://github.com/ClickHouse/ClickHouse/pull/26509) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix lagInFrame for nullable types [#26521](https://github.com/ClickHouse/ClickHouse/pull/26521) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Handle empty testset in 'Functional stateless tests flaky check' [#26552](https://github.com/ClickHouse/ClickHouse/pull/26552) ([Vladimir C](https://github.com/vdimir)). +* Remove some streams. [#26590](https://github.com/ClickHouse/ClickHouse/pull/26590) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test 01622_defaults_for_url_engine [#26617](https://github.com/ClickHouse/ClickHouse/pull/26617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01509_check_many_parallel_quorum_inserts [#26618](https://github.com/ClickHouse/ClickHouse/pull/26618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix one possible cause of tests flakiness [#26619](https://github.com/ClickHouse/ClickHouse/pull/26619) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some code, more C++ way [#26620](https://github.com/ClickHouse/ClickHouse/pull/26620) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Minor bugfix [#26678](https://github.com/ClickHouse/ClickHouse/pull/26678) ([Anton Popov](https://github.com/CurtizJ)). +* copypaste error [#26679](https://github.com/ClickHouse/ClickHouse/pull/26679) ([Denny Crane](https://github.com/den-crane)). +* Fix flaky test `mutation_stuck_after_replace_partition` [#26684](https://github.com/ClickHouse/ClickHouse/pull/26684) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Log errors on integration test listing error in ci-runner [#26691](https://github.com/ClickHouse/ClickHouse/pull/26691) ([Vladimir C](https://github.com/vdimir)). +* more debug checks for window functions [#26701](https://github.com/ClickHouse/ClickHouse/pull/26701) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* record server exit code in fuzzer [#26706](https://github.com/ClickHouse/ClickHouse/pull/26706) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove more streams. [#26713](https://github.com/ClickHouse/ClickHouse/pull/26713) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 01946_test_wrong_host_name_access: Clear DNS in the end [#26715](https://github.com/ClickHouse/ClickHouse/pull/26715) ([Raúl Marín](https://github.com/Algunenano)). +* Setting min_count_to_compile_aggregate_expression fix [#26718](https://github.com/ClickHouse/ClickHouse/pull/26718) ([Maksim Kita](https://github.com/kitaisreal)). +* Compile aggregate functions profile events fix [#26719](https://github.com/ClickHouse/ClickHouse/pull/26719) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix use after free in AsyncDrain connection from S3Cluster [#26731](https://github.com/ClickHouse/ClickHouse/pull/26731) ([Vladimir C](https://github.com/vdimir)). +* Fixed wrong error message in `S3Common` [#26738](https://github.com/ClickHouse/ClickHouse/pull/26738) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Lock mutex before access to std::cerr in clickhouse-benchmark [#26742](https://github.com/ClickHouse/ClickHouse/pull/26742) ([Vladimir C](https://github.com/vdimir)). +* Remove some output streams [#26758](https://github.com/ClickHouse/ClickHouse/pull/26758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#25960](https://github.com/ClickHouse/ClickHouse/issues/25960) (Bolonini/read_from_file) [#26777](https://github.com/ClickHouse/ClickHouse/pull/26777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Heredoc updated tests [#26784](https://github.com/ClickHouse/ClickHouse/pull/26784) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix double unlock in RocksDB [#26786](https://github.com/ClickHouse/ClickHouse/pull/26786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix sqlite engine attach [#26795](https://github.com/ClickHouse/ClickHouse/pull/26795) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove unneeded mutex during connection draining [#26807](https://github.com/ClickHouse/ClickHouse/pull/26807) ([Amos Bird](https://github.com/amosbird)). +* Make sure table is readonly when restarting fails. [#26808](https://github.com/ClickHouse/ClickHouse/pull/26808) ([Amos Bird](https://github.com/amosbird)). +* Check stdout for messages to retry in clickhouse-test [#26817](https://github.com/ClickHouse/ClickHouse/pull/26817) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not start new hedged connection if query was already canceled. [#26820](https://github.com/ClickHouse/ClickHouse/pull/26820) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* Flush LazyOutputFormat on query cancel. [#26828](https://github.com/ClickHouse/ClickHouse/pull/26828) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try fix timeout in functional tests with pytest [#26830](https://github.com/ClickHouse/ClickHouse/pull/26830) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Compile aggregate functions without key [#26845](https://github.com/ClickHouse/ClickHouse/pull/26845) ([Maksim Kita](https://github.com/kitaisreal)). +* Introduce sessions [#26864](https://github.com/ClickHouse/ClickHouse/pull/26864) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix rabbitmq sink [#26871](https://github.com/ClickHouse/ClickHouse/pull/26871) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix keeper bench compilation [#26874](https://github.com/ClickHouse/ClickHouse/pull/26874) ([Raúl Marín](https://github.com/Algunenano)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Maybe fix extremely rare `intersecting parts`. [#26896](https://github.com/ClickHouse/ClickHouse/pull/26896) ([alesapin](https://github.com/alesapin)). +* Try increase diff upper bound [#26897](https://github.com/ClickHouse/ClickHouse/pull/26897) ([Ilya Yatsishin](https://github.com/qoega)). +* Enable Arrow format in Arcadia [#26898](https://github.com/ClickHouse/ClickHouse/pull/26898) ([Vitaly Stoyan](https://github.com/vitstn)). +* Improve test compatibility (00646_url_engine and 01854_HTTP_dict_decompression) [#26915](https://github.com/ClickHouse/ClickHouse/pull/26915) ([Raúl Marín](https://github.com/Algunenano)). +* Update NuRaft [#26916](https://github.com/ClickHouse/ClickHouse/pull/26916) ([alesapin](https://github.com/alesapin)). +* 01921_datatype_date32: Adapt it to work under Pacific/Fiji [#26918](https://github.com/ClickHouse/ClickHouse/pull/26918) ([Raúl Marín](https://github.com/Algunenano)). +* Remove unused files that confuse developers [#26913](https://github.com/ClickHouse/ClickHouse/issues/26913) [#26947](https://github.com/ClickHouse/ClickHouse/pull/26947) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set allow_remote_fs_zero_copy_replication to true by default [#26951](https://github.com/ClickHouse/ClickHouse/pull/26951) ([ianton-ru](https://github.com/ianton-ru)). +* Hold context in HedgedConnections to prevent use-after-free on settings. [#26953](https://github.com/ClickHouse/ClickHouse/pull/26953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix client options in stress test [#26959](https://github.com/ClickHouse/ClickHouse/pull/26959) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix window function partition boundary search [#26960](https://github.com/ClickHouse/ClickHouse/pull/26960) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Print trace from std::terminate exception line-by-line to make it grep easier [#26962](https://github.com/ClickHouse/ClickHouse/pull/26962) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix system.zookeeper_log initialization [#26972](https://github.com/ClickHouse/ClickHouse/pull/26972) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Benchmark script fix [#26974](https://github.com/ClickHouse/ClickHouse/pull/26974) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix 01600_quota_by_forwarded_ip [#26975](https://github.com/ClickHouse/ClickHouse/pull/26975) ([Raúl Marín](https://github.com/Algunenano)). +* 01674_executable_dictionary_implicit_key: executable_dictionary: Use printf [#26978](https://github.com/ClickHouse/ClickHouse/pull/26978) ([Raúl Marín](https://github.com/Algunenano)). +* Remove test_keeper_server usage (for {operation/session}_timeout_ms) [#26984](https://github.com/ClickHouse/ClickHouse/pull/26984) ([Azat Khuzhin](https://github.com/azat)). +* Set insert_quorum_timeout to 1 minute for tests. [#27007](https://github.com/ClickHouse/ClickHouse/pull/27007) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Adjust 00537_quarters to be timezone independent [#27008](https://github.com/ClickHouse/ClickHouse/pull/27008) ([Raúl Marín](https://github.com/Algunenano)). +* Help with [#26424](https://github.com/ClickHouse/ClickHouse/issues/26424) [#27009](https://github.com/ClickHouse/ClickHouse/pull/27009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Attempt to fix flaky 00705_drop_create_merge_tree [#27023](https://github.com/ClickHouse/ClickHouse/pull/27023) ([Raúl Marín](https://github.com/Algunenano)). +* Improved `runner` to use `pytest` keyword expressions [#27026](https://github.com/ClickHouse/ClickHouse/pull/27026) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Improve 01006_simpod_empty_part_single_column_write [#27028](https://github.com/ClickHouse/ClickHouse/pull/27028) ([Raúl Marín](https://github.com/Algunenano)). +* Improved logging of `hwmon` sensor errors in `AsynchronousMetrics` [#27031](https://github.com/ClickHouse/ClickHouse/pull/27031) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix assertions in Replicated database [#27033](https://github.com/ClickHouse/ClickHouse/pull/27033) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Make test 01852_cast_operator independent of timezone [#27037](https://github.com/ClickHouse/ClickHouse/pull/27037) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Moving to TestFlows 1.7.20 that has native support for parallel tests. [#27040](https://github.com/ClickHouse/ClickHouse/pull/27040) ([vzakaznikov](https://github.com/vzakaznikov)). +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix synchronization while updating from the config of an encrypted disk. [#27065](https://github.com/ClickHouse/ClickHouse/pull/27065) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix excessive logging in NuRaft on server shutdown [#27081](https://github.com/ClickHouse/ClickHouse/pull/27081) ([alesapin](https://github.com/alesapin)). +* Fix test_merge_tree_s3_failover with debug build [#27089](https://github.com/ClickHouse/ClickHouse/pull/27089) ([ianton-ru](https://github.com/ianton-ru)). +* Stateless tests: Keep an DNS error free log [#27092](https://github.com/ClickHouse/ClickHouse/pull/27092) ([Raúl Marín](https://github.com/Algunenano)). +* Normalize hostname in stateless tests [#27093](https://github.com/ClickHouse/ClickHouse/pull/27093) ([Amos Bird](https://github.com/amosbird)). +* Try update AMQP-CPP [#27095](https://github.com/ClickHouse/ClickHouse/pull/27095) ([Ilya Yatsishin](https://github.com/qoega)). +* Try update arrow [#27097](https://github.com/ClickHouse/ClickHouse/pull/27097) ([Ilya Yatsishin](https://github.com/qoega)). +* GlobalSubqueriesVisitor external storage check fix [#27131](https://github.com/ClickHouse/ClickHouse/pull/27131) ([Maksim Kita](https://github.com/kitaisreal)). +* Better code around decompression [#27136](https://github.com/ClickHouse/ClickHouse/pull/27136) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add test for parsing maps with integer keys [#27146](https://github.com/ClickHouse/ClickHouse/pull/27146) ([Anton Popov](https://github.com/CurtizJ)). +* Fix arcadia src/Access gtest [#27152](https://github.com/ClickHouse/ClickHouse/pull/27152) ([Ilya Yatsishin](https://github.com/qoega)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Updated readIntTextUnsafe [#27155](https://github.com/ClickHouse/ClickHouse/pull/27155) ([Maksim Kita](https://github.com/kitaisreal)). +* Safer `ReadBufferFromS3` for merges and backports [#27168](https://github.com/ClickHouse/ClickHouse/pull/27168) ([Vladimir Chebotarev](https://github.com/excitoon)). +* properly check the settings in perf test [#27190](https://github.com/ClickHouse/ClickHouse/pull/27190) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Save information about used functions/tables/... into query_log on error [#27194](https://github.com/ClickHouse/ClickHouse/pull/27194) ([Azat Khuzhin](https://github.com/azat)). +* Fix polling of /sys/block [#27195](https://github.com/ClickHouse/ClickHouse/pull/27195) ([Azat Khuzhin](https://github.com/azat)). +* Allow parallel execution of *.sql tests with ReplicatedMergeTree (by using {database} macro) [#27214](https://github.com/ClickHouse/ClickHouse/pull/27214) ([Azat Khuzhin](https://github.com/azat)). +* Fix NLP performance test [#27219](https://github.com/ClickHouse/ClickHouse/pull/27219) ([Nikolay Degterinsky](https://github.com/evillique)). +* Update changelog/README.md [#27221](https://github.com/ClickHouse/ClickHouse/pull/27221) ([filimonov](https://github.com/filimonov)). +* more careful handling of reconnects in fuzzer [#27222](https://github.com/ClickHouse/ClickHouse/pull/27222) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ADDINCL proper fast_float directory [#27224](https://github.com/ClickHouse/ClickHouse/pull/27224) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* DatabaseReplicatedWorker logs_to_keep race fix [#27225](https://github.com/ClickHouse/ClickHouse/pull/27225) ([Maksim Kita](https://github.com/kitaisreal)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Using formatted string literals in clickhouse-test, extracted sort key functions and stacktraces printer [#27228](https://github.com/ClickHouse/ClickHouse/pull/27228) ([Mike Kot](https://github.com/myrrc)). +* Fix polling of /sys/block in case of block devices reopened on error [#27266](https://github.com/ClickHouse/ClickHouse/pull/27266) ([Azat Khuzhin](https://github.com/azat)). +* Remove streams from dicts [#27273](https://github.com/ClickHouse/ClickHouse/pull/27273) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 01099_operators_date_and_timestamp: Use dates that work with all available timezones [#27275](https://github.com/ClickHouse/ClickHouse/pull/27275) ([Raúl Marín](https://github.com/Algunenano)). +* Improve kafka integration test error messages [#27294](https://github.com/ClickHouse/ClickHouse/pull/27294) ([Raúl Marín](https://github.com/Algunenano)). +* Improve 00738_lock_for_inner_table stability [#27300](https://github.com/ClickHouse/ClickHouse/pull/27300) ([Raúl Marín](https://github.com/Algunenano)). +* Add and check system.projection_parts for database filter [#27303](https://github.com/ClickHouse/ClickHouse/pull/27303) ([Azat Khuzhin](https://github.com/azat)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01300_client_save_history_when_terminated_long [#27324](https://github.com/ClickHouse/ClickHouse/pull/27324) ([Raúl Marín](https://github.com/Algunenano)). +* Try update contrib/zlib-ng [#27327](https://github.com/ClickHouse/ClickHouse/pull/27327) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add and check system.mutations for database filter [#27384](https://github.com/ClickHouse/ClickHouse/pull/27384) ([Azat Khuzhin](https://github.com/azat)). +* Correct the key data type used in mapContains [#27423](https://github.com/ClickHouse/ClickHouse/pull/27423) ([Fuwang Hu](https://github.com/fuwhu)). +* Fix tests for WithMergeableStateAfterAggregationAndLimit [#27424](https://github.com/ClickHouse/ClickHouse/pull/27424) ([Azat Khuzhin](https://github.com/azat)). +* Do not miss exceptions from the ThreadPool [#27428](https://github.com/ClickHouse/ClickHouse/pull/27428) ([Azat Khuzhin](https://github.com/azat)). +* Accept error code by error name in client test hints [#27430](https://github.com/ClickHouse/ClickHouse/pull/27430) ([Azat Khuzhin](https://github.com/azat)). +* Added reserve method to Block [#27483](https://github.com/ClickHouse/ClickHouse/pull/27483) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* make it possible to cancel window functions on ctrl+c [#27487](https://github.com/ClickHouse/ClickHouse/pull/27487) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix Nullable const columns in JOIN [#27516](https://github.com/ClickHouse/ClickHouse/pull/27516) ([Vladimir C](https://github.com/vdimir)). +* Fix Logical error: 'Table UUID is not specified in DDL log' [#27521](https://github.com/ClickHouse/ClickHouse/pull/27521) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01236_graphite_mt for random timezones [#27525](https://github.com/ClickHouse/ClickHouse/pull/27525) ([Raúl Marín](https://github.com/Algunenano)). +* Add timeout for integration tests runner [#27535](https://github.com/ClickHouse/ClickHouse/pull/27535) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix polling of /sys/class on errors [#27554](https://github.com/ClickHouse/ClickHouse/pull/27554) ([Azat Khuzhin](https://github.com/azat)). +* fix recalculate QueryMemoryLimitExceeded event error [#27556](https://github.com/ClickHouse/ClickHouse/pull/27556) ([Ben](https://github.com/benbiti)). +* Fix 01961_roaring_memory_tracking for split builds [#27557](https://github.com/ClickHouse/ClickHouse/pull/27557) ([Raúl Marín](https://github.com/Algunenano)). +* Improve the experience of running stateless tests locally [#27561](https://github.com/ClickHouse/ClickHouse/pull/27561) ([Raúl Marín](https://github.com/Algunenano)). +* Fix integration tests [#27562](https://github.com/ClickHouse/ClickHouse/pull/27562) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Dictionaries refactor [#27566](https://github.com/ClickHouse/ClickHouse/pull/27566) ([Maksim Kita](https://github.com/kitaisreal)). +* Less Stopwatch.h [#27569](https://github.com/ClickHouse/ClickHouse/pull/27569) ([filimonov](https://github.com/filimonov)). +* Aggregation temporary disable compilation without key [#27574](https://github.com/ClickHouse/ClickHouse/pull/27574) ([Maksim Kita](https://github.com/kitaisreal)). +* Removed some data streams [#27575](https://github.com/ClickHouse/ClickHouse/pull/27575) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove streams from lv [#27577](https://github.com/ClickHouse/ClickHouse/pull/27577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix ProfileEvents::CompileFunction [#27606](https://github.com/ClickHouse/ClickHouse/pull/27606) ([Ben](https://github.com/benbiti)). +* Refactor mysql format check [#27609](https://github.com/ClickHouse/ClickHouse/pull/27609) ([Raúl Marín](https://github.com/Algunenano)). +* enable part_log by default [#27631](https://github.com/ClickHouse/ClickHouse/pull/27631) ([Denny Crane](https://github.com/den-crane)). +* Remove the remains of ANTLR in the tests [#27637](https://github.com/ClickHouse/ClickHouse/pull/27637) ([Raúl Marín](https://github.com/Algunenano)). +* MV: Improve text logs when doing parallel processing [#27639](https://github.com/ClickHouse/ClickHouse/pull/27639) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test_sqlite_odbc_hashed_dictionary [#27647](https://github.com/ClickHouse/ClickHouse/pull/27647) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#10735](https://github.com/ClickHouse/ClickHouse/issues/10735) [#27677](https://github.com/ClickHouse/ClickHouse/pull/27677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable memory tracking for roaring bitmaps on Mac OS [#27681](https://github.com/ClickHouse/ClickHouse/pull/27681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use only SSE2 in "unbundled" build [#27683](https://github.com/ClickHouse/ClickHouse/pull/27683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash [#27685](https://github.com/ClickHouse/ClickHouse/pull/27685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix stress test in `~CompressedWriteBuffer` [#27686](https://github.com/ClickHouse/ClickHouse/pull/27686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark tests for `DatabaseReplicated` as green [#27688](https://github.com/ClickHouse/ClickHouse/pull/27688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed DenseHashMap, DenseHashSet [#27690](https://github.com/ClickHouse/ClickHouse/pull/27690) ([Maksim Kita](https://github.com/kitaisreal)). +* Map data type parsing tests [#27692](https://github.com/ClickHouse/ClickHouse/pull/27692) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor arrayJoin check on partition expressions [#27733](https://github.com/ClickHouse/ClickHouse/pull/27733) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test 01014_lazy_database_concurrent_recreate_reattach_and_show_tables [#27734](https://github.com/ClickHouse/ClickHouse/pull/27734) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable jemalloc under OSX [#27751](https://github.com/ClickHouse/ClickHouse/pull/27751) ([Raúl Marín](https://github.com/Algunenano)). +* Fix jemalloc under osx (zone_register() had been optimized out again) [#27753](https://github.com/ClickHouse/ClickHouse/pull/27753) ([Azat Khuzhin](https://github.com/azat)). +* Fix intersect/except with limit [#27757](https://github.com/ClickHouse/ClickHouse/pull/27757) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add HTTP string parsing test [#27762](https://github.com/ClickHouse/ClickHouse/pull/27762) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix some tests [#27785](https://github.com/ClickHouse/ClickHouse/pull/27785) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set function divide as suitable for short-circuit in case of Nullable(Decimal) [#27788](https://github.com/ClickHouse/ClickHouse/pull/27788) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unnecessary files [#27789](https://github.com/ClickHouse/ClickHouse/pull/27789) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Mark tests for `DatabaseReplicated` as green" [#27791](https://github.com/ClickHouse/ClickHouse/pull/27791) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove hardening for watches in DDLWorker [#27792](https://github.com/ClickHouse/ClickHouse/pull/27792) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Stateless test: Cleanup leftovers [#27793](https://github.com/ClickHouse/ClickHouse/pull/27793) ([Raúl Marín](https://github.com/Algunenano)). +* Dictionaries key types refactoring [#27795](https://github.com/ClickHouse/ClickHouse/pull/27795) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01822_short_circuit.reference (after merging [#27680](https://github.com/ClickHouse/ClickHouse/issues/27680)) [#27802](https://github.com/ClickHouse/ClickHouse/pull/27802) ([Azat Khuzhin](https://github.com/azat)). +* Proper shutdown global context [#27804](https://github.com/ClickHouse/ClickHouse/pull/27804) ([Amos Bird](https://github.com/amosbird)). +* 01766_todatetime64_no_timezone_arg: Use a date without timezone changes [#27810](https://github.com/ClickHouse/ClickHouse/pull/27810) ([Raúl Marín](https://github.com/Algunenano)). +* Fix setting name "allow_experimental_database_materialized_postgresql" in the error message [#27824](https://github.com/ClickHouse/ClickHouse/pull/27824) ([Denny Crane](https://github.com/den-crane)). +* Fix bug in short-circuit found by fuzzer [#27826](https://github.com/ClickHouse/ClickHouse/pull/27826) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.9.2.17-stable.md b/docs/changelogs/v21.9.2.17-stable.md index 3f132b983c0..c125fab9ed6 100644 --- a/docs/changelogs/v21.9.2.17-stable.md +++ b/docs/changelogs/v21.9.2.17-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.2.17-stable FIXME as compared to v21.9.1.8000-prestable #### Improvement @@ -43,3 +50,15 @@ * Backported in [#28715](https://github.com/ClickHouse/ClickHouse/issues/28715): Add Settings.Names, Settings.Values aliases for system.processes table. [#28685](https://github.com/ClickHouse/ClickHouse/pull/28685) ([Vitaly Orlov](https://github.com/orloffv)). * Backported in [#28744](https://github.com/ClickHouse/ClickHouse/issues/28744): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.9.3.30-stable.md b/docs/changelogs/v21.9.3.30-stable.md index 3b665365668..5318f654f88 100644 --- a/docs/changelogs/v21.9.3.30-stable.md +++ b/docs/changelogs/v21.9.3.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.3.30-stable FIXME as compared to v21.9.2.17-stable #### Improvement @@ -14,3 +21,10 @@ * Backported in [#28927](https://github.com/ClickHouse/ClickHouse/issues/28927): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#29025](https://github.com/ClickHouse/ClickHouse/issues/29025): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.9.4.35-stable.md b/docs/changelogs/v21.9.4.35-stable.md index 8b919ecb268..65ba54de6e8 100644 --- a/docs/changelogs/v21.9.4.35-stable.md +++ b/docs/changelogs/v21.9.4.35-stable.md @@ -1,6 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.9.5.16-stable.md b/docs/changelogs/v21.9.5.16-stable.md index 7287c58064d..305e49a8633 100644 --- a/docs/changelogs/v21.9.5.16-stable.md +++ b/docs/changelogs/v21.9.5.16-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.5.16-stable FIXME as compared to v21.9.4.35-stable #### Improvement @@ -46,3 +53,9 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30185](https://github.com/ClickHouse/ClickHouse/pull/30185) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v21.9.6.24-stable.md b/docs/changelogs/v21.9.6.24-stable.md index f1d097ab646..dbecd4b3634 100644 --- a/docs/changelogs/v21.9.6.24-stable.md +++ b/docs/changelogs/v21.9.6.24-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.6.24-stable FIXME as compared to v21.9.5.16-stable #### New Feature @@ -55,3 +62,10 @@ * Backported in [#32078](https://github.com/ClickHouse/ClickHouse/issues/32078): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). * Backported in [#31907](https://github.com/ClickHouse/ClickHouse/issues/31907): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index c9071422949..a0c6f61d5a6 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.1.2542-prestable FIXME as compared to v21.12.1.9017-prestable #### Backward Incompatible Change @@ -211,6 +218,254 @@ * NO CL ENTRY: 'Added Superwall to adopters list'. [#33573](https://github.com/ClickHouse/ClickHouse/pull/33573) ([Justin Hilliard](https://github.com/jahilliard)). * NO CL ENTRY: 'Revert "Ignore parse failure of opentelemetry header"'. [#33594](https://github.com/ClickHouse/ClickHouse/pull/33594) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Merging [#26074](https://github.com/ClickHouse/ClickHouse/issues/26074) [#26559](https://github.com/ClickHouse/ClickHouse/pull/26559) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Add performance tests to new CI [#31467](https://github.com/ClickHouse/ClickHouse/pull/31467) ([alesapin](https://github.com/alesapin)). +* refactor CI tests [#31882](https://github.com/ClickHouse/ClickHouse/pull/31882) ([Constantine Peresypkin](https://github.com/pkit)). +* Add ability to drain connections synchronously [#31965](https://github.com/ClickHouse/ClickHouse/pull/31965) ([Azat Khuzhin](https://github.com/azat)). +* Add docker container for AWS lambda [#32205](https://github.com/ClickHouse/ClickHouse/pull/32205) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* test for [#28280](https://github.com/ClickHouse/ClickHouse/issues/28280) [#32235](https://github.com/ClickHouse/ClickHouse/pull/32235) ([Denny Crane](https://github.com/den-crane)). +* Rename window functions in WindowView [#32324](https://github.com/ClickHouse/ClickHouse/pull/32324) ([vxider](https://github.com/Vxider)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32389](https://github.com/ClickHouse/ClickHouse/pull/32389) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-local: fix CREATE DATABASE with Atomic engine [#32417](https://github.com/ClickHouse/ClickHouse/pull/32417) ([Azat Khuzhin](https://github.com/azat)). +* Add a test [#16171](https://github.com/ClickHouse/ClickHouse/issues/16171) [#32421](https://github.com/ClickHouse/ClickHouse/pull/32421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix images names in integration tests [#32438](https://github.com/ClickHouse/ClickHouse/pull/32438) ([alesapin](https://github.com/alesapin)). +* Fix build check empty report [#32440](https://github.com/ClickHouse/ClickHouse/pull/32440) ([alesapin](https://github.com/alesapin)). +* Fix ASTFuzzer [#32447](https://github.com/ClickHouse/ClickHouse/pull/32447) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix attaching gdb in tests [#32448](https://github.com/ClickHouse/ClickHouse/pull/32448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix unit tests (ubsan) on master [#32459](https://github.com/ClickHouse/ClickHouse/pull/32459) ([alesapin](https://github.com/alesapin)). +* Remove dependency between integration and functional tests [#32461](https://github.com/ClickHouse/ClickHouse/pull/32461) ([alesapin](https://github.com/alesapin)). +* Add automatic workflow rerun [#32462](https://github.com/ClickHouse/ClickHouse/pull/32462) ([alesapin](https://github.com/alesapin)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update odbc-bridge.md [#32475](https://github.com/ClickHouse/ClickHouse/pull/32475) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test_prometheus_endpoint [#32476](https://github.com/ClickHouse/ClickHouse/pull/32476) ([Vladimir C](https://github.com/vdimir)). +* 02122_parallel_formatting: Address grep binary warnings [#32477](https://github.com/ClickHouse/ClickHouse/pull/32477) ([Raúl Marín](https://github.com/Algunenano)). +* 01950_kill_large_group_by_query: Increase timeout [#32479](https://github.com/ClickHouse/ClickHouse/pull/32479) ([Raúl Marín](https://github.com/Algunenano)). +* Fix backport workflow [#32485](https://github.com/ClickHouse/ClickHouse/pull/32485) ([alesapin](https://github.com/alesapin)). +* Make 01675_distributed_bytes_to_delay_insert less flaky (use http over client) [#32492](https://github.com/ClickHouse/ClickHouse/pull/32492) ([Azat Khuzhin](https://github.com/azat)). +* Fix 02050_client_profile_events flakiness [#32493](https://github.com/ClickHouse/ClickHouse/pull/32493) ([Azat Khuzhin](https://github.com/azat)). +* Fix integration tests docker images path [#32494](https://github.com/ClickHouse/ClickHouse/pull/32494) ([alesapin](https://github.com/alesapin)). +* Split long tests into multiple checks [#32496](https://github.com/ClickHouse/ClickHouse/pull/32496) ([alesapin](https://github.com/alesapin)). +* Add test for clickhouse-client history navigation [#32497](https://github.com/ClickHouse/ClickHouse/pull/32497) ([Amos Bird](https://github.com/amosbird)). +* Rename window-view function to time window function [#32498](https://github.com/ClickHouse/ClickHouse/pull/32498) ([vxider](https://github.com/Vxider)). +* Disable flaky tests with Window View [#32500](https://github.com/ClickHouse/ClickHouse/pull/32500) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky window view tests [#32503](https://github.com/ClickHouse/ClickHouse/pull/32503) ([vxider](https://github.com/Vxider)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to fix attaching gdb 2 [#32511](https://github.com/ClickHouse/ClickHouse/pull/32511) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: use basename of the test for *.sh tests [#32533](https://github.com/ClickHouse/ClickHouse/pull/32533) ([Azat Khuzhin](https://github.com/azat)). +* Fix processing initial table (--table/stdin) in clickhouse-local [#32534](https://github.com/ClickHouse/ClickHouse/pull/32534) ([Azat Khuzhin](https://github.com/azat)). +* Suppress UBSan errors for avg() function [#32535](https://github.com/ClickHouse/ClickHouse/pull/32535) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR for MATERIALIZED VIEW over table functions (i.e. numbers()) [#32571](https://github.com/ClickHouse/ClickHouse/pull/32571) ([Azat Khuzhin](https://github.com/azat)). +* Improve quota's end-of-interval calculations. [#32575](https://github.com/ClickHouse/ClickHouse/pull/32575) ([Vitaly Baranov](https://github.com/vitlibar)). +* remove unused headers in test [#32578](https://github.com/ClickHouse/ClickHouse/pull/32578) ([Bharat Nallan](https://github.com/bharatnc)). +* Remove arcadia build support [#32580](https://github.com/ClickHouse/ClickHouse/pull/32580) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup perf test runner [#32582](https://github.com/ClickHouse/ClickHouse/pull/32582) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] perf: do not fail in case of slow queries (to avoid hiding possible issues) [#32583](https://github.com/ClickHouse/ClickHouse/pull/32583) ([Azat Khuzhin](https://github.com/azat)). +* Add test for issue 28316 [#32585](https://github.com/ClickHouse/ClickHouse/pull/32585) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bad test [#32571](https://github.com/ClickHouse/ClickHouse/issues/32571) [#32588](https://github.com/ClickHouse/ClickHouse/pull/32588) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix perf test `reinterpret_as` [#32589](https://github.com/ClickHouse/ClickHouse/pull/32589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix indecent error message [#32591](https://github.com/ClickHouse/ClickHouse/pull/32591) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix strange code in TCPHandler [#32592](https://github.com/ClickHouse/ClickHouse/pull/32592) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* CI: upload logs if task failed [#32618](https://github.com/ClickHouse/ClickHouse/pull/32618) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: avoid failing with UNKNOWN status in some cases [#32625](https://github.com/ClickHouse/ClickHouse/pull/32625) ([Azat Khuzhin](https://github.com/azat)). +* avoid inconsistent state in Block [#32641](https://github.com/ClickHouse/ClickHouse/pull/32641) ([gulige](https://github.com/gulige)). +* Rerun workflows more times [#32642](https://github.com/ClickHouse/ClickHouse/pull/32642) ([alesapin](https://github.com/alesapin)). +* Remove flaky check from master and split asan test [#32645](https://github.com/ClickHouse/ClickHouse/pull/32645) ([alesapin](https://github.com/alesapin)). +* Split database replicated [#32650](https://github.com/ClickHouse/ClickHouse/pull/32650) ([alesapin](https://github.com/alesapin)). +* Improve exceptions usage in access control [#32662](https://github.com/ClickHouse/ClickHouse/pull/32662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32667](https://github.com/ClickHouse/ClickHouse/pull/32667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add test_s3_zero_copy_concurrent_merge [#32694](https://github.com/ClickHouse/ClickHouse/pull/32694) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Check trusted contributors in lowercase [#32696](https://github.com/ClickHouse/ClickHouse/pull/32696) ([Vladimir C](https://github.com/vdimir)). +* Mention ClickHouse CLA in CONTRIBUTING.md [#32697](https://github.com/ClickHouse/ClickHouse/pull/32697) ([Ivan Blinkov](https://github.com/blinkov)). +* Rerun release workflows [#32701](https://github.com/ClickHouse/ClickHouse/pull/32701) ([alesapin](https://github.com/alesapin)). +* plausible analytic blog post [#32719](https://github.com/ClickHouse/ClickHouse/pull/32719) ([Tom Risse](https://github.com/flickerbox-tom)). +* Update backport.py [#32720](https://github.com/ClickHouse/ClickHouse/pull/32720) ([Kruglov Pavel](https://github.com/Avogar)). +* Pass -no-pie to the linker call to disable it [#32731](https://github.com/ClickHouse/ClickHouse/pull/32731) ([Raúl Marín](https://github.com/Algunenano)). +* Add severity table to support case form [#32736](https://github.com/ClickHouse/ClickHouse/pull/32736) ([Cody Baker](https://github.com/codyrobert)). +* tests/integration: fix wait_start()/start_clickhouse() [#32739](https://github.com/ClickHouse/ClickHouse/pull/32739) ([Azat Khuzhin](https://github.com/azat)). +* tests/integration: fix waiting of mysql client container [#32740](https://github.com/ClickHouse/ClickHouse/pull/32740) ([Azat Khuzhin](https://github.com/azat)). +* Better control build artifacts [#32745](https://github.com/ClickHouse/ClickHouse/pull/32745) ([alesapin](https://github.com/alesapin)). +* Fix race in skipping index of type `hypothesis` [#32756](https://github.com/ClickHouse/ClickHouse/pull/32756) ([Anton Popov](https://github.com/CurtizJ)). +* Fix build performance [#32766](https://github.com/ClickHouse/ClickHouse/pull/32766) ([alesapin](https://github.com/alesapin)). +* Add test for issue 26912 [#32771](https://github.com/ClickHouse/ClickHouse/pull/32771) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for CAST to named tuple just in case. [#32776](https://github.com/ClickHouse/ClickHouse/pull/32776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance build [#32796](https://github.com/ClickHouse/ClickHouse/pull/32796) ([alesapin](https://github.com/alesapin)). +* Trying another stress servers [#32798](https://github.com/ClickHouse/ClickHouse/pull/32798) ([alesapin](https://github.com/alesapin)). +* More diagnostics in gdb script [#32801](https://github.com/ClickHouse/ClickHouse/pull/32801) ([Alexander Tokmakov](https://github.com/tavplubix)). +* CacheDictionary dictionary source access race fix [#32805](https://github.com/ClickHouse/ClickHouse/pull/32805) ([Maksim Kita](https://github.com/kitaisreal)). +* Update NuRaft [#32808](https://github.com/ClickHouse/ClickHouse/pull/32808) ([alesapin](https://github.com/alesapin)). +* Add retries to curl in performance tests. [#32809](https://github.com/ClickHouse/ClickHouse/pull/32809) ([alesapin](https://github.com/alesapin)). +* Made Azure blob storage to obey ENABLE_LIBRARIES flag [#32815](https://github.com/ClickHouse/ClickHouse/pull/32815) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove recursive submodules [#32820](https://github.com/ClickHouse/ClickHouse/pull/32820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* test for summap nullable(0) [#32822](https://github.com/ClickHouse/ClickHouse/pull/32822) ([Denny Crane](https://github.com/den-crane)). +* Update CHANGELOG.md [#32823](https://github.com/ClickHouse/ClickHouse/pull/32823) ([Denny Crane](https://github.com/den-crane)). +* Make `test_storage_s3` less flaky [#32825](https://github.com/ClickHouse/ClickHouse/pull/32825) ([Azat Khuzhin](https://github.com/azat)). +* Fix error message for reading from url [#32826](https://github.com/ClickHouse/ClickHouse/pull/32826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert docs release [#32827](https://github.com/ClickHouse/ClickHouse/pull/32827) ([alesapin](https://github.com/alesapin)). +* Fix docs release one more time [#32828](https://github.com/ClickHouse/ClickHouse/pull/32828) ([alesapin](https://github.com/alesapin)). +* Trigger also for workflows changes [#32830](https://github.com/ClickHouse/ClickHouse/pull/32830) ([alesapin](https://github.com/alesapin)). +* Fix docs release one more time [#32831](https://github.com/ClickHouse/ClickHouse/pull/32831) ([alesapin](https://github.com/alesapin)). +* Add dispatch event to docs release [#32832](https://github.com/ClickHouse/ClickHouse/pull/32832) ([alesapin](https://github.com/alesapin)). +* Remove repository before checkout [#32834](https://github.com/ClickHouse/ClickHouse/pull/32834) ([alesapin](https://github.com/alesapin)). +* Fix release branches filter [#32837](https://github.com/ClickHouse/ClickHouse/pull/32837) ([alesapin](https://github.com/alesapin)). +* StorageLiveView fix function style [#32838](https://github.com/ClickHouse/ClickHouse/pull/32838) ([Maksim Kita](https://github.com/kitaisreal)). +* Try fix flaky test: order for test_s3_zero_copy_replication [#32846](https://github.com/ClickHouse/ClickHouse/pull/32846) ([Vladimir C](https://github.com/vdimir)). +* Update TRUSTED_CONTIBUTORS [#32850](https://github.com/ClickHouse/ClickHouse/pull/32850) ([Vladimir C](https://github.com/vdimir)). +* Followup docs check workflow [#32857](https://github.com/ClickHouse/ClickHouse/pull/32857) ([alesapin](https://github.com/alesapin)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use a proper syntax for multiline env in GITHUB_ENV [#32864](https://github.com/ClickHouse/ClickHouse/pull/32864) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix multiline SSH_KEY in GITHUB_ENV file [#32865](https://github.com/ClickHouse/ClickHouse/pull/32865) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* warm up for test_kafka_json_as_string_no_kdc [#32885](https://github.com/ClickHouse/ClickHouse/pull/32885) ([Ilya Golshtein](https://github.com/ilejn)). +* Rename files and code from BlobStorage to AzureBlobStorage [#32886](https://github.com/ClickHouse/ClickHouse/pull/32886) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add dockerhub proxy to CI [#32905](https://github.com/ClickHouse/ClickHouse/pull/32905) ([alesapin](https://github.com/alesapin)). +* Update 01069_window_view_proc_tumble_watch.py [#32908](https://github.com/ClickHouse/ClickHouse/pull/32908) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update test azure_blob_storage::test_table_manipulations [#32909](https://github.com/ClickHouse/ClickHouse/pull/32909) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More retries in perf tests. [#32910](https://github.com/ClickHouse/ClickHouse/pull/32910) ([alesapin](https://github.com/alesapin)). +* fix wrong testcase in gtest_Parser [#32912](https://github.com/ClickHouse/ClickHouse/pull/32912) ([SuperDJY](https://github.com/cmsxbc)). +* Disable percpu arena in jemalloc in case of non deterministic CPU count [#32916](https://github.com/ClickHouse/ClickHouse/pull/32916) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Add replxx/jemalloc into fasttest build of clickhouse [#32923](https://github.com/ClickHouse/ClickHouse/pull/32923) ([Azat Khuzhin](https://github.com/azat)). +* tests/integration/cleanup_environment: fix obtaining PYTEST_CLEANUP_CONTAINERS [#32931](https://github.com/ClickHouse/ClickHouse/pull/32931) ([Azat Khuzhin](https://github.com/azat)). +* fix typo [#32936](https://github.com/ClickHouse/ClickHouse/pull/32936) ([Denny Crane](https://github.com/den-crane)). +* Fix build issue related to azure blob storage [another try] [#32948](https://github.com/ClickHouse/ClickHouse/pull/32948) ([Amos Bird](https://github.com/amosbird)). +* test for base64encode_trailing_bytes [#31797](https://github.com/ClickHouse/ClickHouse/issues/31797) [#32954](https://github.com/ClickHouse/ClickHouse/pull/32954) ([Denny Crane](https://github.com/den-crane)). +* simplify looping in S2 functions [#32960](https://github.com/ClickHouse/ClickHouse/pull/32960) ([Bharat Nallan](https://github.com/bharatnc)). +* Sync release CI branches in master [#32967](https://github.com/ClickHouse/ClickHouse/pull/32967) ([alesapin](https://github.com/alesapin)). +* Loops remove postfix increment [#32970](https://github.com/ClickHouse/ClickHouse/pull/32970) ([Maksim Kita](https://github.com/kitaisreal)). +* Containers iteration fix erase [#32972](https://github.com/ClickHouse/ClickHouse/pull/32972) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#32400](https://github.com/ClickHouse/ClickHouse/issues/32400) [#32974](https://github.com/ClickHouse/ClickHouse/pull/32974) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Decrease log level for some s3 messages. [#32979](https://github.com/ClickHouse/ClickHouse/pull/32979) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Trying to add woboq to new CI [#32983](https://github.com/ClickHouse/ClickHouse/pull/32983) ([alesapin](https://github.com/alesapin)). +* RabbitMQ tests fix [#32985](https://github.com/ClickHouse/ClickHouse/pull/32985) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy [#32987](https://github.com/ClickHouse/ClickHouse/pull/32987) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo [#32992](https://github.com/ClickHouse/ClickHouse/pull/32992) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix weird permission on log directories [#32994](https://github.com/ClickHouse/ClickHouse/pull/32994) ([Håvard Kvålen](https://github.com/havardk)). +* Add Keeper Jepsen check to new CI [#32998](https://github.com/ClickHouse/ClickHouse/pull/32998) ([alesapin](https://github.com/alesapin)). +* Add backport workflow rerun [#33005](https://github.com/ClickHouse/ClickHouse/pull/33005) ([alesapin](https://github.com/alesapin)). +* Remove the possibility of adding columns with table overrides [#33017](https://github.com/ClickHouse/ClickHouse/pull/33017) ([Stig Bakken](https://github.com/stigsb)). +* Merge [#33023](https://github.com/ClickHouse/ClickHouse/issues/33023) [#33027](https://github.com/ClickHouse/ClickHouse/pull/33027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix cron expression [#33035](https://github.com/ClickHouse/ClickHouse/pull/33035) ([alesapin](https://github.com/alesapin)). +* Make `can be tested` great again [#33036](https://github.com/ClickHouse/ClickHouse/pull/33036) ([alesapin](https://github.com/alesapin)). +* Fix exception in azure write buffer desctructor [#33039](https://github.com/ClickHouse/ClickHouse/pull/33039) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add manual run for woboq [#33042](https://github.com/ClickHouse/ClickHouse/pull/33042) ([alesapin](https://github.com/alesapin)). +* Update Cassandra driver [#33043](https://github.com/ClickHouse/ClickHouse/pull/33043) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Decrease default timeout for integration test [#33044](https://github.com/ClickHouse/ClickHouse/pull/33044) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix merge [#33021](https://github.com/ClickHouse/ClickHouse/issues/33021) [#33049](https://github.com/ClickHouse/ClickHouse/pull/33049) ([Nikolay Degterinsky](https://github.com/evillique)). +* Update banner for january 20 release webinar [#33052](https://github.com/ClickHouse/ClickHouse/pull/33052) ([Cody Baker](https://github.com/codyrobert)). +* Update harmful library [#33058](https://github.com/ClickHouse/ClickHouse/pull/33058) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33025](https://github.com/ClickHouse/ClickHouse/issues/33025) [#33063](https://github.com/ClickHouse/ClickHouse/pull/33063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add alternative LRUCache version [#33098](https://github.com/ClickHouse/ClickHouse/pull/33098) ([lgbo](https://github.com/lgbo-ustc)). +* Fix test that was dependent on time zone [#33122](https://github.com/ClickHouse/ClickHouse/pull/33122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang tidy [#33131](https://github.com/ClickHouse/ClickHouse/pull/33131) ([alesapin](https://github.com/alesapin)). +* Fix PVS check [#33135](https://github.com/ClickHouse/ClickHouse/pull/33135) ([alesapin](https://github.com/alesapin)). +* Upload build artifact in case of build failures [#33136](https://github.com/ClickHouse/ClickHouse/pull/33136) ([alesapin](https://github.com/alesapin)). +* Fix woboq codebrowser [#33137](https://github.com/ClickHouse/ClickHouse/pull/33137) ([alesapin](https://github.com/alesapin)). +* Longer timeout for server start in fuzzer [#33138](https://github.com/ClickHouse/ClickHouse/pull/33138) ([alesapin](https://github.com/alesapin)). +* Fix azure blob storage tests failures [#33140](https://github.com/ClickHouse/ClickHouse/pull/33140) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix functions with sparse arguments in case when result type doesn't support sparse serialization [#33146](https://github.com/ClickHouse/ClickHouse/pull/33146) ([Anton Popov](https://github.com/CurtizJ)). +* Add pretty print unknow event [#33148](https://github.com/ClickHouse/ClickHouse/pull/33148) ([alesapin](https://github.com/alesapin)). +* Better pr info for unknown events [#33149](https://github.com/ClickHouse/ClickHouse/pull/33149) ([alesapin](https://github.com/alesapin)). +* Better jepsen [#33150](https://github.com/ClickHouse/ClickHouse/pull/33150) ([alesapin](https://github.com/alesapin)). +* Debugging PR info [#33151](https://github.com/ClickHouse/ClickHouse/pull/33151) ([alesapin](https://github.com/alesapin)). +* Fix shared hermetic builds on Arch linux [#33153](https://github.com/ClickHouse/ClickHouse/pull/33153) ([Azat Khuzhin](https://github.com/azat)). +* Fix ccache with ENABLE_CHECK_HEAVY_BUILDS (ccache 4.0 and 4.1 only affected) [#33154](https://github.com/ClickHouse/ClickHouse/pull/33154) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup trash from Kafka, HDFS and Azure [#33160](https://github.com/ClickHouse/ClickHouse/pull/33160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix Date32 test [#33161](https://github.com/ClickHouse/ClickHouse/pull/33161) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make test_input_format_parallel_parsing_memory_tracking less flaky [#33163](https://github.com/ClickHouse/ClickHouse/pull/33163) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_async_drain_connection flakiness [#33164](https://github.com/ClickHouse/ClickHouse/pull/33164) ([Azat Khuzhin](https://github.com/azat)). +* Fix clang-tidy [#33168](https://github.com/ClickHouse/ClickHouse/pull/33168) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01622_defaults_for_url_engine [#33169](https://github.com/ClickHouse/ClickHouse/pull/33169) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* materialized postgresql better startup [#33177](https://github.com/ClickHouse/ClickHouse/pull/33177) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible uncaught exception in clickhouse-local [#33189](https://github.com/ClickHouse/ClickHouse/pull/33189) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix tests with event_time/event_date = today(), and add a style check [#33198](https://github.com/ClickHouse/ClickHouse/pull/33198) ([Azat Khuzhin](https://github.com/azat)). +* Fix 01370_client_autocomplete_word_break_characters test logic [#33202](https://github.com/ClickHouse/ClickHouse/pull/33202) ([Azat Khuzhin](https://github.com/azat)). +* More debug in jepsen test [#33207](https://github.com/ClickHouse/ClickHouse/pull/33207) ([alesapin](https://github.com/alesapin)). +* Fix some flaky integration tests [#33215](https://github.com/ClickHouse/ClickHouse/pull/33215) ([alesapin](https://github.com/alesapin)). +* Remove old trash [#33217](https://github.com/ClickHouse/ClickHouse/pull/33217) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix cancel lambda [#33222](https://github.com/ClickHouse/ClickHouse/pull/33222) ([alesapin](https://github.com/alesapin)). +* Fix keeper log messages [#33224](https://github.com/ClickHouse/ClickHouse/pull/33224) ([alesapin](https://github.com/alesapin)). +* Fix clang tidy [#33235](https://github.com/ClickHouse/ClickHouse/pull/33235) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 01650 [#33250](https://github.com/ClickHouse/ClickHouse/pull/33250) ([alesapin](https://github.com/alesapin)). +* Add retries for github api [#33252](https://github.com/ClickHouse/ClickHouse/pull/33252) ([alesapin](https://github.com/alesapin)). +* Add retries to AST fuzzer download and fix flaky test. [#33256](https://github.com/ClickHouse/ClickHouse/pull/33256) ([alesapin](https://github.com/alesapin)). +* Do not ignore eof in expect tests [#33263](https://github.com/ClickHouse/ClickHouse/pull/33263) ([Azat Khuzhin](https://github.com/azat)). +* Fix parsing symbols from resources (for shared builds) [#33264](https://github.com/ClickHouse/ClickHouse/pull/33264) ([Azat Khuzhin](https://github.com/azat)). +* test for [#24410](https://github.com/ClickHouse/ClickHouse/issues/24410) [#33265](https://github.com/ClickHouse/ClickHouse/pull/33265) ([Denny Crane](https://github.com/den-crane)). +* Wait for RabbitMQ container to actually start when it was restarted in test on purpose [#33266](https://github.com/ClickHouse/ClickHouse/pull/33266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Mark max_alter_threads as obsolete [#33268](https://github.com/ClickHouse/ClickHouse/pull/33268) ([Denny Crane](https://github.com/den-crane)). +* Fix azure tests flackyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Test for [#26920](https://github.com/ClickHouse/ClickHouse/issues/26920) [#33272](https://github.com/ClickHouse/ClickHouse/pull/33272) ([Denny Crane](https://github.com/den-crane)). +* Fix test_storage_kafka failures by adjusting retention.ms [#33278](https://github.com/ClickHouse/ClickHouse/pull/33278) ([Azat Khuzhin](https://github.com/azat)). +* Disable FunctionConvertFromString::canBeExecutedOnDefaultArguments [#33286](https://github.com/ClickHouse/ClickHouse/pull/33286) ([Vladimir C](https://github.com/vdimir)). +* Add IAST::QueryKind enum and use it in query limiter [#33294](https://github.com/ClickHouse/ClickHouse/pull/33294) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix clang tidy 3 [#33296](https://github.com/ClickHouse/ClickHouse/pull/33296) ([alesapin](https://github.com/alesapin)). +* Fix jepsen check [#33304](https://github.com/ClickHouse/ClickHouse/pull/33304) ([alesapin](https://github.com/alesapin)). +* Add test for broken connection [#33305](https://github.com/ClickHouse/ClickHouse/pull/33305) ([nvartolomei](https://github.com/nvartolomei)). +* Fix launcher not being defined for ccache < 4.0 and slightly cleanup [#33306](https://github.com/ClickHouse/ClickHouse/pull/33306) ([nvartolomei](https://github.com/nvartolomei)). +* Return early if azure blob storage is not used, otherwise it might with fail with irrelevant errors [#33307](https://github.com/ClickHouse/ClickHouse/pull/33307) ([nvartolomei](https://github.com/nvartolomei)). +* Minor refactor of read buffers with compression [#33318](https://github.com/ClickHouse/ClickHouse/pull/33318) ([Nikolay Degterinsky](https://github.com/evillique)). +* Rename Committed data part state to Active [#33327](https://github.com/ClickHouse/ClickHouse/pull/33327) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Tiny follow-up to pr 33098 [#33334](https://github.com/ClickHouse/ClickHouse/pull/33334) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix inner table parser in window view [#33340](https://github.com/ClickHouse/ClickHouse/pull/33340) ([vxider](https://github.com/Vxider)). +* Cover CustomSeparated/Template formats for Kafka [#33343](https://github.com/ClickHouse/ClickHouse/pull/33343) ([Azat Khuzhin](https://github.com/azat)). +* update copyright in docs [#33353](https://github.com/ClickHouse/ClickHouse/pull/33353) ([Bharat Nallan](https://github.com/bharatnc)). +* UserDefinedExecutableFunction fix exception [#33362](https://github.com/ClickHouse/ClickHouse/pull/33362) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to merge [#23230](https://github.com/ClickHouse/ClickHouse/issues/23230) [#33368](https://github.com/ClickHouse/ClickHouse/pull/33368) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Print alter command names in EXPLAIN AST [#33383](https://github.com/ClickHouse/ClickHouse/pull/33383) ([Stig Bakken](https://github.com/stigsb)). +* DictionaryStructure fixes [#33388](https://github.com/ClickHouse/ClickHouse/pull/33388) ([Maksim Kita](https://github.com/kitaisreal)). +* tests/stress: add core dumps into artifacts [#33389](https://github.com/ClickHouse/ClickHouse/pull/33389) ([Azat Khuzhin](https://github.com/azat)). +* Update success.html [#33397](https://github.com/ClickHouse/ClickHouse/pull/33397) ([Adri Fernandez](https://github.com/iladriano)). +* Remove obsolete and unsupported Docker Builder [#33345](https://github.com/ClickHouse/ClickHouse/issues/33345) [#33399](https://github.com/ClickHouse/ClickHouse/pull/33399) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe improve robustness of the "query profiler" test [#33422](https://github.com/ClickHouse/ClickHouse/pull/33422) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix misleading log message in StorageWindowView [#33432](https://github.com/ClickHouse/ClickHouse/pull/33432) ([flynn](https://github.com/ucasfl)). +* Preparation for [#33167](https://github.com/ClickHouse/ClickHouse/issues/33167). [#33444](https://github.com/ClickHouse/ClickHouse/pull/33444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rerun docs release check as well [#33447](https://github.com/ClickHouse/ClickHouse/pull/33447) ([alesapin](https://github.com/alesapin)). +* Update AsynchronousReadIndirectBufferFromRemoteFS.h [#33452](https://github.com/ClickHouse/ClickHouse/pull/33452) ([xiedeyantu](https://github.com/xiedeyantu)). +* Tests visualizer better [#33453](https://github.com/ClickHouse/ClickHouse/pull/33453) ([Dmitry Khorkin](https://github.com/DimaAmega)). +* Function monthName small fix [#33464](https://github.com/ClickHouse/ClickHouse/pull/33464) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for operator priorities in SQL [#33487](https://github.com/ClickHouse/ClickHouse/pull/33487) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add new team members to company page [#33506](https://github.com/ClickHouse/ClickHouse/pull/33506) ([Cody Baker](https://github.com/codyrobert)). +* fix variable name in h3 misc funcs [#33510](https://github.com/ClickHouse/ClickHouse/pull/33510) ([Bharat Nallan](https://github.com/bharatnc)). +* H3 functions bad arguments possible crash fix [#33512](https://github.com/ClickHouse/ClickHouse/pull/33512) ([Maksim Kita](https://github.com/kitaisreal)). +* Build with ENABLE_TESTS fix [#33513](https://github.com/ClickHouse/ClickHouse/pull/33513) ([Maksim Kita](https://github.com/kitaisreal)). +* IAST QueryKind style fix [#33514](https://github.com/ClickHouse/ClickHouse/pull/33514) ([Maksim Kita](https://github.com/kitaisreal)). +* Don't print exception twice in client in case of exception in parallel parsing [#33524](https://github.com/ClickHouse/ClickHouse/pull/33524) ([Kruglov Pavel](https://github.com/Avogar)). +* HashTable constant fix [#33525](https://github.com/ClickHouse/ClickHouse/pull/33525) ([Maksim Kita](https://github.com/kitaisreal)). +* Add admixer blog post [#33528](https://github.com/ClickHouse/ClickHouse/pull/33528) ([Cody Baker](https://github.com/codyrobert)). +* Fix clickhouse local interactive exception case [#33535](https://github.com/ClickHouse/ClickHouse/pull/33535) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix parsing queries with FROM INFILE statement again [#33556](https://github.com/ClickHouse/ClickHouse/pull/33556) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logic for upload upload_master_static_binaries [#33560](https://github.com/ClickHouse/ClickHouse/pull/33560) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Don't upload splitted binaries to builds.clickhouse.com [#33562](https://github.com/ClickHouse/ClickHouse/pull/33562) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix minor bug in TTL and flaky test [#33564](https://github.com/ClickHouse/ClickHouse/pull/33564) ([Alexander Tokmakov](https://github.com/tavplubix)). +* skip MySQLCreateRewritten unit testsuite when enable_mysql is off [#33577](https://github.com/ClickHouse/ClickHouse/pull/33577) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Fix test `02156_storage_merge_prewhere` [#33580](https://github.com/ClickHouse/ClickHouse/pull/33580) ([Anton Popov](https://github.com/CurtizJ)). +* Update mongodb.md [#33585](https://github.com/ClickHouse/ClickHouse/pull/33585) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Restore existing static builds links [#33597](https://github.com/ClickHouse/ClickHouse/pull/33597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix pylint for run_check.py [#33600](https://github.com/ClickHouse/ClickHouse/pull/33600) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flacky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make ZooKeeper client better interpret keeper server connection reject [#33602](https://github.com/ClickHouse/ClickHouse/pull/33602) ([alesapin](https://github.com/alesapin)). +* Fix broken workflow dependencies [#33608](https://github.com/ClickHouse/ClickHouse/pull/33608) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Force rebuild images in CI [#33609](https://github.com/ClickHouse/ClickHouse/pull/33609) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* YetAnother workflow fix [#33610](https://github.com/ClickHouse/ClickHouse/pull/33610) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Metrics lambda: fix deletion [#33616](https://github.com/ClickHouse/ClickHouse/pull/33616) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add an exception message example [#33623](https://github.com/ClickHouse/ClickHouse/pull/33623) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix two flaky integration tests [#33625](https://github.com/ClickHouse/ClickHouse/pull/33625) ([alesapin](https://github.com/alesapin)). +* Update version to 22.1 not 21.13 [#33626](https://github.com/ClickHouse/ClickHouse/pull/33626) ([alesapin](https://github.com/alesapin)). +* do not construct std::string if there is no error [#33628](https://github.com/ClickHouse/ClickHouse/pull/33628) ([Sergei Trifonov](https://github.com/serxa)). +* Fix test 00900_long_parquet_load [#33629](https://github.com/ClickHouse/ClickHouse/pull/33629) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix error codes order [#33633](https://github.com/ClickHouse/ClickHouse/pull/33633) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix s3 integration tests [#33641](https://github.com/ClickHouse/ClickHouse/pull/33641) ([Kruglov Pavel](https://github.com/Avogar)). +* test for [#33592](https://github.com/ClickHouse/ClickHouse/issues/33592) [#33658](https://github.com/ClickHouse/ClickHouse/pull/33658) ([Denny Crane](https://github.com/den-crane)). +* Update "gosu" version [#33660](https://github.com/ClickHouse/ClickHouse/pull/33660) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01814_distributed_push_down_limit flakiness [#33662](https://github.com/ClickHouse/ClickHouse/pull/33662) ([Azat Khuzhin](https://github.com/azat)). +* Add amosbird to trusted user. [#33663](https://github.com/ClickHouse/ClickHouse/pull/33663) ([Amos Bird](https://github.com/amosbird)). +* Fix pr ci restart [#33667](https://github.com/ClickHouse/ClickHouse/pull/33667) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + #### New Feature / New Tool * Tool for collecting diagnostics data. [#33175](https://github.com/ClickHouse/ClickHouse/pull/33175) ([Alexander Burmak](https://github.com/Alex-Burmak)). diff --git a/docs/changelogs/v22.1.2.2-stable.md b/docs/changelogs/v22.1.2.2-stable.md index 450c640bc5e..de2dc6e3cee 100644 --- a/docs/changelogs/v22.1.2.2-stable.md +++ b/docs/changelogs/v22.1.2.2-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.2.2-stable FIXME as compared to v22.1.1.2542-prestable diff --git a/docs/changelogs/v22.1.3.7-stable.md b/docs/changelogs/v22.1.3.7-stable.md index f8bb8821031..72c7a9cc5da 100644 --- a/docs/changelogs/v22.1.3.7-stable.md +++ b/docs/changelogs/v22.1.3.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.3.7-stable FIXME as compared to v22.1.2.2-stable #### Improvement diff --git a/docs/changelogs/v22.1.4.30-stable.md b/docs/changelogs/v22.1.4.30-stable.md index 1ea56131481..7fa295cdcb0 100644 --- a/docs/changelogs/v22.1.4.30-stable.md +++ b/docs/changelogs/v22.1.4.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.4.30-stable FIXME as compared to v22.1.3.7-stable #### Build/Testing/Packaging Improvement diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index 26ac24a7778..379fb8988c8 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.1.2139-prestable FIXME as compared to v22.1.1.2542-prestable #### New Feature @@ -208,3 +215,164 @@ * NO CL ENTRY: 'Add support agreement page and snippets.'. [#34512](https://github.com/ClickHouse/ClickHouse/pull/34512) ([Tom Risse](https://github.com/flickerbox-tom)). * NO CL ENTRY: 'Add Gigasheet to adopters'. [#34589](https://github.com/ClickHouse/ClickHouse/pull/34589) ([Brian Hunter](https://github.com/bjhunter)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix status of SKIPPED integration tests (was incorrectly marked as ERROR) [#33162](https://github.com/ClickHouse/ClickHouse/pull/33162) ([Azat Khuzhin](https://github.com/azat)). +* Client interactive suggest (extract info from CREATE queries) [#33201](https://github.com/ClickHouse/ClickHouse/pull/33201) ([Azat Khuzhin](https://github.com/azat)). +* Fix flushing of in-memory parts [#33234](https://github.com/ClickHouse/ClickHouse/pull/33234) ([Anton Popov](https://github.com/CurtizJ)). +* improvements to tests for h3kRing and h3ToChildren funcs [#33311](https://github.com/ClickHouse/ClickHouse/pull/33311) ([Bharat Nallan](https://github.com/bharatnc)). +* Simplify different block structure (i.e. after ALTER) support for Buffer [#33324](https://github.com/ClickHouse/ClickHouse/pull/33324) ([Azat Khuzhin](https://github.com/azat)). +* Dictionary rename fix [#33526](https://github.com/ClickHouse/ClickHouse/pull/33526) ([Maksim Kita](https://github.com/kitaisreal)). +* RFC: Split headers, move SystemLog into module, more forward declarations [#33534](https://github.com/ClickHouse/ClickHouse/pull/33534) ([Azat Khuzhin](https://github.com/azat)). +* add check for h3 empty column arguments [#33552](https://github.com/ClickHouse/ClickHouse/pull/33552) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix testflows tests [#33575](https://github.com/ClickHouse/ClickHouse/pull/33575) ([Vitaly Baranov](https://github.com/vitlibar)). +* Ignore parse failure of opentelemetry header. Another try. [#33595](https://github.com/ClickHouse/ClickHouse/pull/33595) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix mongo no_auth compose [#33634](https://github.com/ClickHouse/ClickHouse/pull/33634) ([Ilya Yatsishin](https://github.com/qoega)). +* AsynchronousMetrics: Ignore inaccessible sensors [#33639](https://github.com/ClickHouse/ClickHouse/pull/33639) ([Raúl Marín](https://github.com/Algunenano)). +* Slightly reduce memory usage for parsing collections (by using move ctor) [#33665](https://github.com/ClickHouse/ClickHouse/pull/33665) ([Azat Khuzhin](https://github.com/azat)). +* Merge [#33563](https://github.com/ClickHouse/ClickHouse/issues/33563) [#33669](https://github.com/ClickHouse/ClickHouse/pull/33669) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release 22.1 [#33673](https://github.com/ClickHouse/ClickHouse/pull/33673) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix sumIf rewrite [#33677](https://github.com/ClickHouse/ClickHouse/pull/33677) ([flynn](https://github.com/ucasfl)). +* DictionarySourceCoordinator update interface [#33682](https://github.com/ClickHouse/ClickHouse/pull/33682) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix LDAP and Kerberos config handling [#33689](https://github.com/ClickHouse/ClickHouse/pull/33689) ([Denis Glazachev](https://github.com/traceon)). +* Add some helping comments for API endpoints [#33700](https://github.com/ClickHouse/ClickHouse/pull/33700) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix hive tests [#33703](https://github.com/ClickHouse/ClickHouse/pull/33703) ([Kseniia Sumarokova](https://github.com/kssenii)). +* H3 remove functions degsToRads, radsToDegs [#33707](https://github.com/ClickHouse/ClickHouse/pull/33707) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove unused variable [#33731](https://github.com/ClickHouse/ClickHouse/pull/33731) ([vxider](https://github.com/Vxider)). +* Add test issue_31009 [#33739](https://github.com/ClickHouse/ClickHouse/pull/33739) ([Vladimir C](https://github.com/vdimir)). +* Cleanup build: .gitignore more debian directories, libpqxx-cmake without configure_file [#33742](https://github.com/ClickHouse/ClickHouse/pull/33742) ([Ilya Yatsishin](https://github.com/qoega)). +* Better exception text on suspicious broken parts [#33743](https://github.com/ClickHouse/ClickHouse/pull/33743) ([alesapin](https://github.com/alesapin)). +* Fix release_branches workflow for some cases [#33744](https://github.com/ClickHouse/ClickHouse/pull/33744) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add new team members to company page [#33755](https://github.com/ClickHouse/ClickHouse/pull/33755) ([Cody Baker](https://github.com/codyrobert)). +* Remove duplicated header [#33760](https://github.com/ClickHouse/ClickHouse/pull/33760) ([vxider](https://github.com/Vxider)). +* Hotfix of missing header [#33765](https://github.com/ClickHouse/ClickHouse/pull/33765) ([Amos Bird](https://github.com/amosbird)). +* rewrite bitHammingDistance with FunctionBinaryArithmetic [#33772](https://github.com/ClickHouse/ClickHouse/pull/33772) ([flynn](https://github.com/ucasfl)). +* Use tb64senc for base64Decode on aarch64 [#33779](https://github.com/ClickHouse/ClickHouse/pull/33779) ([Vladimir C](https://github.com/vdimir)). +* Support USE_* build flags in tests' tags [#33780](https://github.com/ClickHouse/ClickHouse/pull/33780) ([Vladimir C](https://github.com/vdimir)). +* Use workflow names in approve lambda [#33789](https://github.com/ClickHouse/ClickHouse/pull/33789) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Dictionaries remove unnecessary copy of keys during read [#33791](https://github.com/ClickHouse/ClickHouse/pull/33791) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove debug logging from TableFunctionFile [#33796](https://github.com/ClickHouse/ClickHouse/pull/33796) ([Kruglov Pavel](https://github.com/Avogar)). +* RangeHashedDictionary handle invalid intervals [#33827](https://github.com/ClickHouse/ClickHouse/pull/33827) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries added Date32 type support [#33831](https://github.com/ClickHouse/ClickHouse/pull/33831) ([Maksim Kita](https://github.com/kitaisreal)). +* TypeId better naming [#33832](https://github.com/ClickHouse/ClickHouse/pull/33832) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionsConversion fix typo [#33841](https://github.com/ClickHouse/ClickHouse/pull/33841) ([Maksim Kita](https://github.com/kitaisreal)). +* More cmake external modules cleanups [#33842](https://github.com/ClickHouse/ClickHouse/pull/33842) ([Azat Khuzhin](https://github.com/azat)). +* Remove harmful code and check what will happen [#33844](https://github.com/ClickHouse/ClickHouse/pull/33844) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix DESCRIBE TABLE query formatting [#33846](https://github.com/ClickHouse/ClickHouse/pull/33846) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix build with ENABLE_EXAMPLES [#33848](https://github.com/ClickHouse/ClickHouse/pull/33848) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Trademark Policy Page [#33851](https://github.com/ClickHouse/ClickHouse/pull/33851) ([Cody Baker](https://github.com/codyrobert)). +* updates to min2 and max2 funcs [#33852](https://github.com/ClickHouse/ClickHouse/pull/33852) ([Bharat Nallan](https://github.com/bharatnc)). +* Use correct logging level [#33857](https://github.com/ClickHouse/ClickHouse/pull/33857) ([李扬](https://github.com/taiyang-li)). +* Fix zookeeper library dependency from interpreters (by marking library STATIC) [#33860](https://github.com/ClickHouse/ClickHouse/pull/33860) ([Azat Khuzhin](https://github.com/azat)). +* FunctionBase64Conversion warning fix [#33863](https://github.com/ClickHouse/ClickHouse/pull/33863) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary ATTACH, DETACH added test [#33870](https://github.com/ClickHouse/ClickHouse/pull/33870) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix perf comparison rerun [#33872](https://github.com/ClickHouse/ClickHouse/pull/33872) ([alesapin](https://github.com/alesapin)). +* Improve postgresql integration test [#33880](https://github.com/ClickHouse/ClickHouse/pull/33880) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix cmake for mac. [#33882](https://github.com/ClickHouse/ClickHouse/pull/33882) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* SQLUserDefinedFunctions invalid lambda additional fixes [#33889](https://github.com/ClickHouse/ClickHouse/pull/33889) ([Maksim Kita](https://github.com/kitaisreal)). +* Update upcoming and past webinar links on homepage [#33890](https://github.com/ClickHouse/ClickHouse/pull/33890) ([Cody Baker](https://github.com/codyrobert)). +* disable animation on docs menu [#33903](https://github.com/ClickHouse/ClickHouse/pull/33903) ([SuperDJY](https://github.com/cmsxbc)). +* Add more retries in CI for AWS "spot" instances [#33907](https://github.com/ClickHouse/ClickHouse/pull/33907) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionMathUnary remove macro usage [#33916](https://github.com/ClickHouse/ClickHouse/pull/33916) ([Maksim Kita](https://github.com/kitaisreal)). +* explicitly checkAndGetColumn for remaining H3 funcs [#33921](https://github.com/ClickHouse/ClickHouse/pull/33921) ([Bharat Nallan](https://github.com/bharatnc)). +* SQLUserDefinedFunctions invalid lambda additional cases [#33924](https://github.com/ClickHouse/ClickHouse/pull/33924) ([Maksim Kita](https://github.com/kitaisreal)). +* when not DEBUG set USE_DEBUG_HELPERS OFF [#33925](https://github.com/ClickHouse/ClickHouse/pull/33925) ([Ben](https://github.com/benbiti)). +* Fix test_replica_is_active flaky test [#33926](https://github.com/ClickHouse/ClickHouse/pull/33926) ([alesapin](https://github.com/alesapin)). +* Fix bug in keeper which can lead to inconsistent snapshots [#33941](https://github.com/ClickHouse/ClickHouse/pull/33941) ([alesapin](https://github.com/alesapin)). +* Fix keeper data dumper build [#33942](https://github.com/ClickHouse/ClickHouse/pull/33942) ([alesapin](https://github.com/alesapin)). +* Remove MAKE_STATIC_LIBRARIES (in favor of USE_STATIC_LIBRARIES) [#33946](https://github.com/ClickHouse/ClickHouse/pull/33946) ([Azat Khuzhin](https://github.com/azat)). +* Revert glibc compatibility (via .symver) in favor of hermetic build (bundled libc) [#33950](https://github.com/ClickHouse/ClickHouse/pull/33950) ([Azat Khuzhin](https://github.com/azat)). +* add c++expr script for C++ one-liners [#33962](https://github.com/ClickHouse/ClickHouse/pull/33962) ([Sergei Trifonov](https://github.com/serxa)). +* docker: fix root squashed data dirs [#33963](https://github.com/ClickHouse/ClickHouse/pull/33963) ([Constantine Peresypkin](https://github.com/pkit)). +* Disable parallel run for 00985_merge_stack_overflow [#33976](https://github.com/ClickHouse/ClickHouse/pull/33976) ([Azat Khuzhin](https://github.com/azat)). +* Cancel lambda [#33990](https://github.com/ClickHouse/ClickHouse/pull/33990) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* reduce signal_pipe_buf_size [#33996](https://github.com/ClickHouse/ClickHouse/pull/33996) ([save-my-heart](https://github.com/save-my-heart)). +* make systemd to use EnvironmentFile [#34024](https://github.com/ClickHouse/ClickHouse/pull/34024) ([Denny Crane](https://github.com/den-crane)). +* CurrentlyExecuting: Require mutex usage explicitly [#34034](https://github.com/ClickHouse/ClickHouse/pull/34034) ([Raúl Marín](https://github.com/Algunenano)). +* Add symlinks to keeper [#34042](https://github.com/ClickHouse/ClickHouse/pull/34042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DecimalPaddedPODArray [#34052](https://github.com/ClickHouse/ClickHouse/pull/34052) ([Maksim Kita](https://github.com/kitaisreal)). +* Less flaky test_inconsistent_parts_if_drop_while_replica_not_active [#34057](https://github.com/ClickHouse/ClickHouse/pull/34057) ([Raúl Marín](https://github.com/Algunenano)). +* Small test for `FINAL` [#34058](https://github.com/ClickHouse/ClickHouse/pull/34058) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Slightly optimize ColumnArray::get()/operator[] (by using reserve over resize) [#34074](https://github.com/ClickHouse/ClickHouse/pull/34074) ([Azat Khuzhin](https://github.com/azat)). +* Tiny cleanup of AggregateFunctionSimpleState/AggregateFunctionState [#34075](https://github.com/ClickHouse/ClickHouse/pull/34075) ([Azat Khuzhin](https://github.com/azat)). +* Fix builds [#34090](https://github.com/ClickHouse/ClickHouse/pull/34090) ([Alexander Tokmakov](https://github.com/tavplubix)). +* add c++expr script examples [#34112](https://github.com/ClickHouse/ClickHouse/pull/34112) ([Sergei Trifonov](https://github.com/serxa)). +* Make clickhouse-diagnostics also work for altinity release [#34116](https://github.com/ClickHouse/ClickHouse/pull/34116) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Small improvement in schema inference from stdin in local [#34117](https://github.com/ClickHouse/ClickHouse/pull/34117) ([Kruglov Pavel](https://github.com/Avogar)). +* Slightly optimize Array/Tuple/Map [#34126](https://github.com/ClickHouse/ClickHouse/pull/34126) ([Azat Khuzhin](https://github.com/azat)). +* Sort block refactoring [#34143](https://github.com/ClickHouse/ClickHouse/pull/34143) ([Maksim Kita](https://github.com/kitaisreal)). +* pdqsort performance check [#34145](https://github.com/ClickHouse/ClickHouse/pull/34145) ([Maksim Kita](https://github.com/kitaisreal)). +* Revert [#33957](https://github.com/ClickHouse/ClickHouse/issues/33957) [#34146](https://github.com/ClickHouse/ClickHouse/pull/34146) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add func tests run with s3 [#34153](https://github.com/ClickHouse/ClickHouse/pull/34153) ([alesapin](https://github.com/alesapin)). +* Close [#10197](https://github.com/ClickHouse/ClickHouse/issues/10197) [#34159](https://github.com/ClickHouse/ClickHouse/pull/34159) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cmake leftovers cleanup [#34167](https://github.com/ClickHouse/ClickHouse/pull/34167) ([Azat Khuzhin](https://github.com/azat)). +* bitsetsort peformance check [#34175](https://github.com/ClickHouse/ClickHouse/pull/34175) ([Maksim Kita](https://github.com/kitaisreal)). +* Add authorisation for dockerhub proxy container [#34183](https://github.com/ClickHouse/ClickHouse/pull/34183) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue [#34189](https://github.com/ClickHouse/ClickHouse/pull/34189) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some perf tests [#34191](https://github.com/ClickHouse/ClickHouse/pull/34191) ([Kruglov Pavel](https://github.com/Avogar)). +* Clean up: insert_deduplication_token setting for INSERT statement [#34192](https://github.com/ClickHouse/ClickHouse/pull/34192) ([Igor Nikonov](https://github.com/devcrafter)). +* Add func tests run with s3 and fix several bugs [#34215](https://github.com/ClickHouse/ClickHouse/pull/34215) ([alesapin](https://github.com/alesapin)). +* Revert "Revert "Add pool to WriteBufferFromS3"" [#34219](https://github.com/ClickHouse/ClickHouse/pull/34219) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Revert "Revert "Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue"" [#34221](https://github.com/ClickHouse/ClickHouse/pull/34221) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add const to make clang-tidy happy [#34222](https://github.com/ClickHouse/ClickHouse/pull/34222) ([Vitaly Baranov](https://github.com/vitlibar)). +* Bump fmtlib from 7.0.0 to 8.1.1 [#34223](https://github.com/ClickHouse/ClickHouse/pull/34223) ([Azat Khuzhin](https://github.com/azat)). +* Add submodule minizip [#34226](https://github.com/ClickHouse/ClickHouse/pull/34226) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update list-versions.sh, update version_date.tsv [#34240](https://github.com/ClickHouse/ClickHouse/pull/34240) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add missing fmt::runtime() in MergeTreeBackgroundExecutor (fixes the build) [#34245](https://github.com/ClickHouse/ClickHouse/pull/34245) ([Azat Khuzhin](https://github.com/azat)). +* Update clickhouse-keeper.md [#34264](https://github.com/ClickHouse/ClickHouse/pull/34264) ([Andrew](https://github.com/andycol)). +* print query id when using `--interactive` with `--queries-file` in client/local [#34265](https://github.com/ClickHouse/ClickHouse/pull/34265) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update and add new team members [#34268](https://github.com/ClickHouse/ClickHouse/pull/34268) ([Cody Baker](https://github.com/codyrobert)). +* Add a test for shebang [#34274](https://github.com/ClickHouse/ClickHouse/pull/34274) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fractional_progress_bar test [#34282](https://github.com/ClickHouse/ClickHouse/pull/34282) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor fixes for [#34267](https://github.com/ClickHouse/ClickHouse/issues/34267) [#34284](https://github.com/ClickHouse/ClickHouse/pull/34284) ([Anton Popov](https://github.com/CurtizJ)). +* Add support policy page content. [#34309](https://github.com/ClickHouse/ClickHouse/pull/34309) ([Tom Risse](https://github.com/flickerbox-tom)). +* Probably fix data race in WriteBufferFromS3 destructor. [#34316](https://github.com/ClickHouse/ClickHouse/pull/34316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Decrease severity for "Reading ... ranges ..." log message to Trace [#34319](https://github.com/ClickHouse/ClickHouse/pull/34319) ([Azat Khuzhin](https://github.com/azat)). +* use LowCardinality type for _file and _path in some Storages, continue of [#34317](https://github.com/ClickHouse/ClickHouse/issues/34317) [#34333](https://github.com/ClickHouse/ClickHouse/pull/34333) ([flynn](https://github.com/ucasfl)). +* Function mapPopulateSeries added additional performance test [#34339](https://github.com/ClickHouse/ClickHouse/pull/34339) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for propagating OpenTelemetry context via gRPC protocol [#34341](https://github.com/ClickHouse/ClickHouse/pull/34341) ([Vitaly Baranov](https://github.com/vitlibar)). +* explicitly check and get columns for s2 funcs [#34344](https://github.com/ClickHouse/ClickHouse/pull/34344) ([Bharat Nallan](https://github.com/bharatnc)). +* Small improvements [#34351](https://github.com/ClickHouse/ClickHouse/pull/34351) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix USE database for clickhouse-local [#34357](https://github.com/ClickHouse/ClickHouse/pull/34357) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy issue [#34365](https://github.com/ClickHouse/ClickHouse/pull/34365) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Delete empty file DateOrDateTimeFunctionsConvertion.cpp [#34371](https://github.com/ClickHouse/ClickHouse/pull/34371) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix consecutive backward seeks in seekable read buffers [#34376](https://github.com/ClickHouse/ClickHouse/pull/34376) ([Anton Popov](https://github.com/CurtizJ)). +* Enable one more check for clang-tidy [#34388](https://github.com/ClickHouse/ClickHouse/pull/34388) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Method called on already moved [#34398](https://github.com/ClickHouse/ClickHouse/pull/34398) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Fix wrong destruction order in CreatingSetsTransform. [#34406](https://github.com/ClickHouse/ClickHouse/pull/34406) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Split and rename compression fields in gRPC [#34408](https://github.com/ClickHouse/ClickHouse/pull/34408) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixing test_storage_postgresql [#34410](https://github.com/ClickHouse/ClickHouse/pull/34410) ([Kseniia Sumarokova](https://github.com/kssenii)). +* asynchronous_inserts engine AsynchronousInserts -> SystemAsynchronousInserts [#34429](https://github.com/ClickHouse/ClickHouse/pull/34429) ([filimonov](https://github.com/filimonov)). +* clang-tidy move fix build [#34431](https://github.com/ClickHouse/ClickHouse/pull/34431) ([Maksim Kita](https://github.com/kitaisreal)). +* `static-files-disk-uploader`: add a mode to create symlinks [#34432](https://github.com/ClickHouse/ClickHouse/pull/34432) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bug in URL engine [#34448](https://github.com/ClickHouse/ClickHouse/pull/34448) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix style [#34450](https://github.com/ClickHouse/ClickHouse/pull/34450) ([alesapin](https://github.com/alesapin)). +* Added test 33734 [#34454](https://github.com/ClickHouse/ClickHouse/pull/34454) ([Maksim Kita](https://github.com/kitaisreal)). +* Update http_max_tries setting default [#34457](https://github.com/ClickHouse/ClickHouse/pull/34457) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use `cpp_bin_float_double` in `set_multiplier` for `wide_integer_from_builtin` for aarch64 [#34463](https://github.com/ClickHouse/ClickHouse/pull/34463) ([Vladimir C](https://github.com/vdimir)). +* test for [#13907](https://github.com/ClickHouse/ClickHouse/issues/13907) toColumnTypeName_toLowCardinality_const [#34471](https://github.com/ClickHouse/ClickHouse/pull/34471) ([Denny Crane](https://github.com/den-crane)). +* Remove invalid IOS setting for RocksDB CMAKE to fix Apple M1 build [#34472](https://github.com/ClickHouse/ClickHouse/pull/34472) ([Geoff Genz](https://github.com/genzgd)). +* clang-tidy reported potential chance for divide by zero exception [#34473](https://github.com/ClickHouse/ClickHouse/pull/34473) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* accessing nested_column after already moved to data [#34475](https://github.com/ClickHouse/ClickHouse/pull/34475) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Avoid unnecessary copying of `Settings` [#34476](https://github.com/ClickHouse/ClickHouse/pull/34476) ([Anton Popov](https://github.com/CurtizJ)). +* Update buildPushingToViewsChain.h [#34515](https://github.com/ClickHouse/ClickHouse/pull/34515) ([William.Walliams.Wong](https://github.com/wangzhen11aaa)). +* More gdb introspection on CI [#34517](https://github.com/ClickHouse/ClickHouse/pull/34517) ([Azat Khuzhin](https://github.com/azat)). +* add BeforeLambdaBody to .clang-format [#34533](https://github.com/ClickHouse/ClickHouse/pull/34533) ([Vladimir C](https://github.com/vdimir)). +* Remove very old cruft [#34547](https://github.com/ClickHouse/ClickHouse/pull/34547) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tiny fixes for client/local suggestions [#34550](https://github.com/ClickHouse/ClickHouse/pull/34550) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix gtest_archive_reader_and_writer in case of !USE_MINIZIP [#34554](https://github.com/ClickHouse/ClickHouse/pull/34554) ([Azat Khuzhin](https://github.com/azat)). +* TableFunctionFile added performance test [#34555](https://github.com/ClickHouse/ClickHouse/pull/34555) ([Maksim Kita](https://github.com/kitaisreal)). +* check and get columns in geoToH3 func [#34557](https://github.com/ClickHouse/ClickHouse/pull/34557) ([Bharat Nallan](https://github.com/bharatnc)). +* try fix data race in StorageLog [#34558](https://github.com/ClickHouse/ClickHouse/pull/34558) ([flynn](https://github.com/ucasfl)). +* Postpone a bit nightly builds to the least loaded time [#34569](https://github.com/ClickHouse/ClickHouse/pull/34569) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Always remove unused actions from addMissingDefaults [#34577](https://github.com/ClickHouse/ClickHouse/pull/34577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix indentation in ARRAY JOIN formatting [#34578](https://github.com/ClickHouse/ClickHouse/pull/34578) ([Azat Khuzhin](https://github.com/azat)). +* Fix deadlock in OvercommitTracker [#34591](https://github.com/ClickHouse/ClickHouse/pull/34591) ([Dmitry Novik](https://github.com/novikd)). +* ASTCreateQuery: Remove usused `tables` member [#34610](https://github.com/ClickHouse/ClickHouse/pull/34610) ([Raúl Marín](https://github.com/Algunenano)). +* Add test for [#19222](https://github.com/ClickHouse/ClickHouse/issues/19222) [#34615](https://github.com/ClickHouse/ClickHouse/pull/34615) ([Raúl Marín](https://github.com/Algunenano)). +* Attempt to fix freeBSD build [#34617](https://github.com/ClickHouse/ClickHouse/pull/34617) ([Raúl Marín](https://github.com/Algunenano)). +* Fix quadratic complexity while adding subcolumns [#34623](https://github.com/ClickHouse/ClickHouse/pull/34623) ([Anton Popov](https://github.com/CurtizJ)). +* Fix typo in welchttest.md [#34634](https://github.com/ClickHouse/ClickHouse/pull/34634) ([achimbab](https://github.com/achimbab)). +* Improve the release.py script [#34659](https://github.com/ClickHouse/ClickHouse/pull/34659) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable memory overcommit if timeout is not set [#34663](https://github.com/ClickHouse/ClickHouse/pull/34663) ([Dmitry Novik](https://github.com/novikd)). + diff --git a/docs/changelogs/v22.2.2.1-stable.md b/docs/changelogs/v22.2.2.1-stable.md index c9158290753..0ea5be33c94 100644 --- a/docs/changelogs/v22.2.2.1-stable.md +++ b/docs/changelogs/v22.2.2.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.2.1-stable FIXME as compared to v22.2.1.2139-prestable diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index 90c0d22d570..2eb8a6b63cc 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -1,6 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.3.5-stable FIXME as compared to v22.2.2.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#34848](https://github.com/ClickHouse/ClickHouse/issues/34848): Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix using host/port from config for client [#34791](https://github.com/ClickHouse/ClickHouse/pull/34791) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index 03e81bd1808..0058396d634 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.1.1262-prestable FIXME as compared to v22.2.1.2139-prestable #### Backward Incompatible Change @@ -65,7 +72,7 @@ * Add setting to lower column case when reading parquet/ORC file. [#35145](https://github.com/ClickHouse/ClickHouse/pull/35145) ([shuchaome](https://github.com/shuchaome)). * Do not retry non-rertiable errors. Closes [#35161](https://github.com/ClickHouse/ClickHouse/issues/35161). [#35172](https://github.com/ClickHouse/ClickHouse/pull/35172) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added disk_name to system.part_log. [#35178](https://github.com/ClickHouse/ClickHouse/pull/35178) ([Artyom Yurkov](https://github.com/Varinara)). -* Currently,ClickHouse validates hosts defined under for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Currently, ClickHouse validates hosts defined under for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). * Sometimes it is not enough for us to distinguish queries hierachy only by is_initial_query in system.query_log and system.processes. So distributed_depth is needed. [#35207](https://github.com/ClickHouse/ClickHouse/pull/35207) ([李扬](https://github.com/taiyang-li)). * Support test mode for clickhouse-local. [#35264](https://github.com/ClickHouse/ClickHouse/pull/35264) ([Kseniia Sumarokova](https://github.com/kssenii)). * Return const for function getMacro if not in distributed query. Close [#34727](https://github.com/ClickHouse/ClickHouse/issues/34727). [#35289](https://github.com/ClickHouse/ClickHouse/pull/35289) ([李扬](https://github.com/taiyang-li)). @@ -144,3 +151,124 @@ * NO CL ENTRY: 'Revert "Change timezone in Docker"'. [#35243](https://github.com/ClickHouse/ClickHouse/pull/35243) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Fix 00900_long_parquet_load"'. [#35301](https://github.com/ClickHouse/ClickHouse/pull/35301) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* [RFC] Update jemalloc to 5.3RC [#33057](https://github.com/ClickHouse/ClickHouse/pull/33057) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless setting experimental_query_deduplication_send_all_part_uuids [#34387](https://github.com/ClickHouse/ClickHouse/pull/34387) ([nvartolomei](https://github.com/nvartolomei)). +* Apply join_use_nulls on types before join [#34529](https://github.com/ClickHouse/ClickHouse/pull/34529) ([Vladimir C](https://github.com/vdimir)). +* Adjust 01681_arg_min_max_if_fix [#34612](https://github.com/ClickHouse/ClickHouse/pull/34612) ([Vladimir C](https://github.com/vdimir)). +* Check overflow in addSeconds/Minues/etc functions [#34619](https://github.com/ClickHouse/ClickHouse/pull/34619) ([Vladimir C](https://github.com/vdimir)). +* Update version after release [#34677](https://github.com/ClickHouse/ClickHouse/pull/34677) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix `positionUTF8` on aarch64 [#34683](https://github.com/ClickHouse/ClickHouse/pull/34683) ([Vladimir C](https://github.com/vdimir)). +* Update version_date.tsv after v22.2.2.1-stable [#34685](https://github.com/ClickHouse/ClickHouse/pull/34685) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix ZooKepper paths in zero_copy_schema_converter.py [#34686](https://github.com/ClickHouse/ClickHouse/pull/34686) ([ianton-ru](https://github.com/ianton-ru)). +* Changelog for version 22.2 [#34687](https://github.com/ClickHouse/ClickHouse/pull/34687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow up to PR [#33698](https://github.com/ClickHouse/ClickHouse/issues/33698) [#34689](https://github.com/ClickHouse/ClickHouse/pull/34689) ([Vladimir C](https://github.com/vdimir)). +* Always update ProfileEvents (even on exceptions) [#34690](https://github.com/ClickHouse/ClickHouse/pull/34690) ([nvartolomei](https://github.com/nvartolomei)). +* Function encodeURLComponent minor fixes [#34695](https://github.com/ClickHouse/ClickHouse/pull/34695) ([Maksim Kita](https://github.com/kitaisreal)). +* CMake llvm add status message if not used [#34698](https://github.com/ClickHouse/ClickHouse/pull/34698) ([Maksim Kita](https://github.com/kitaisreal)). +* Adding noexcept for move constructor SharedContextHolder [#34699](https://github.com/ClickHouse/ClickHouse/pull/34699) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add new team members to company page [#34701](https://github.com/ClickHouse/ClickHouse/pull/34701) ([Cody Baker](https://github.com/codyrobert)). +* Fix flaky 00502_custom_partitioning_local [#34714](https://github.com/ClickHouse/ClickHouse/pull/34714) ([Azat Khuzhin](https://github.com/azat)). +* Workaround for a bug in NuRaft library [#34715](https://github.com/ClickHouse/ClickHouse/pull/34715) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible memory_tracker use-after-free (for async s3 writes) for merges/mutations [#34717](https://github.com/ClickHouse/ClickHouse/pull/34717) ([Azat Khuzhin](https://github.com/azat)). +* Add taiyang-li to trusted contributors [#34729](https://github.com/ClickHouse/ClickHouse/pull/34729) ([Vladimir C](https://github.com/vdimir)). +* rename name in ci check [#34730](https://github.com/ClickHouse/ClickHouse/pull/34730) ([flynn](https://github.com/ucasfl)). +* Performance tests fix H3 [#34731](https://github.com/ClickHouse/ClickHouse/pull/34731) ([Maksim Kita](https://github.com/kitaisreal)). +* Enable async writes to S3. [#34734](https://github.com/ClickHouse/ClickHouse/pull/34734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add a patch release for stable/lts packages to release.py [#34740](https://github.com/ClickHouse/ClickHouse/pull/34740) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Print only total profile events for --profile-events-delay-ms=-1 [#34749](https://github.com/ClickHouse/ClickHouse/pull/34749) ([Azat Khuzhin](https://github.com/azat)). +* Fix `parallel_reading_from_replicas` with `clickhouse-bechmark` [#34751](https://github.com/ClickHouse/ClickHouse/pull/34751) ([Azat Khuzhin](https://github.com/azat)). +* Submodules are mandatory [#34753](https://github.com/ClickHouse/ClickHouse/pull/34753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changelog for version 22.2 [#34758](https://github.com/ClickHouse/ClickHouse/pull/34758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Sort additional_html_urls in test report in CI [#34770](https://github.com/ClickHouse/ClickHouse/pull/34770) ([Vladimir C](https://github.com/vdimir)). +* Fix undefined __pthread_mutex_lock/unlock for glibc 2.34+/DISABLE_HERMETIC_BUILD [#34771](https://github.com/ClickHouse/ClickHouse/pull/34771) ([Azat Khuzhin](https://github.com/azat)). +* jemalloc: fix includes order to avoid overlaps [#34783](https://github.com/ClickHouse/ClickHouse/pull/34783) ([Azat Khuzhin](https://github.com/azat)). +* Fix using host/port from config for client [#34791](https://github.com/ClickHouse/ClickHouse/pull/34791) ([Vladimir C](https://github.com/vdimir)). +* Update links for 22.2 and 22.3 release webinars [#34814](https://github.com/ClickHouse/ClickHouse/pull/34814) ([Cody Baker](https://github.com/codyrobert)). +* Add opensee blog post [#34817](https://github.com/ClickHouse/ClickHouse/pull/34817) ([Cody Baker](https://github.com/codyrobert)). +* Try remove trash from StorageFileLog [#34819](https://github.com/ClickHouse/ClickHouse/pull/34819) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix generating system.build_options [#34823](https://github.com/ClickHouse/ClickHouse/pull/34823) ([Azat Khuzhin](https://github.com/azat)). +* Comment output header for team keys [#34828](https://github.com/ClickHouse/ClickHouse/pull/34828) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix untuple condition in IN function [#34836](https://github.com/ClickHouse/ClickHouse/pull/34836) ([Vladimir C](https://github.com/vdimir)). +* Improve certificate reloader [#34887](https://github.com/ClickHouse/ClickHouse/pull/34887) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Update DiskS3.cpp [#34889](https://github.com/ClickHouse/ClickHouse/pull/34889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Clang tidy improvements [#34893](https://github.com/ClickHouse/ClickHouse/pull/34893) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version_date.tsv after v22.2.3.5-stable [#34898](https://github.com/ClickHouse/ClickHouse/pull/34898) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Add check_prerequisites to release.py [#34901](https://github.com/ClickHouse/ClickHouse/pull/34901) ([Vladimir C](https://github.com/vdimir)). +* Fix filelog storage data paths [#34912](https://github.com/ClickHouse/ClickHouse/pull/34912) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update version_date.tsv after v22.1.4.30-stable [#34913](https://github.com/ClickHouse/ClickHouse/pull/34913) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix alignment for aligned_alloc() to fix build for glibc2.35+ [#34919](https://github.com/ClickHouse/ClickHouse/pull/34919) ([Azat Khuzhin](https://github.com/azat)). +* Stop processing multiqueries in clickhouse-client/local on SIGINT [#34923](https://github.com/ClickHouse/ClickHouse/pull/34923) ([Azat Khuzhin](https://github.com/azat)). +* Fix some code comments style [#34935](https://github.com/ClickHouse/ClickHouse/pull/34935) ([hongbin](https://github.com/xlwh)). +* Adjust couple of tests for aarch64 [#34951](https://github.com/ClickHouse/ClickHouse/pull/34951) ([Vladimir C](https://github.com/vdimir)). +* Add benchmarks [#34960](https://github.com/ClickHouse/ClickHouse/pull/34960) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Catch exception in ~WriteBufferFromS3 [#34963](https://github.com/ClickHouse/ClickHouse/pull/34963) ([Vladimir C](https://github.com/vdimir)). +* Disable memory checking test with thread sanitizer [#34964](https://github.com/ClickHouse/ClickHouse/pull/34964) ([alesapin](https://github.com/alesapin)). +* Print number of available logical cores in configuration [#34972](https://github.com/ClickHouse/ClickHouse/pull/34972) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix possible "nmalloc >= ndalloc" jemalloc assertion [#34973](https://github.com/ClickHouse/ClickHouse/pull/34973) ([Azat Khuzhin](https://github.com/azat)). +* Add some benchmarks [#34974](https://github.com/ClickHouse/ClickHouse/pull/34974) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Fix typo [#34975](https://github.com/ClickHouse/ClickHouse/pull/34975) ([Nikita Evsyukov](https://github.com/NikitaEvs)). +* Fix signal-unsafe TSan report in client [#34988](https://github.com/ClickHouse/ClickHouse/pull/34988) ([Azat Khuzhin](https://github.com/azat)). +* Fix non-MergeTree engines for system.*_log [#34989](https://github.com/ClickHouse/ClickHouse/pull/34989) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_storage_postgresql::test_concurrent_queries [#34991](https://github.com/ClickHouse/ClickHouse/pull/34991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix race between INSERT async_insert=1 and system.asynchronous_inserts [#34992](https://github.com/ClickHouse/ClickHouse/pull/34992) ([Azat Khuzhin](https://github.com/azat)). +* Add option to clickhouse-test to skip aarch64 build [#34995](https://github.com/ClickHouse/ClickHouse/pull/34995) ([Vladimir C](https://github.com/vdimir)). +* clickhouse obfuscator aarch64 fix [#35005](https://github.com/ClickHouse/ClickHouse/pull/35005) ([Vladimir C](https://github.com/vdimir)). +* Fix tutorial link [#35014](https://github.com/ClickHouse/ClickHouse/pull/35014) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* ExecutableUDF function deterministic in scope of query fix [#35018](https://github.com/ClickHouse/ClickHouse/pull/35018) ([Maksim Kita](https://github.com/kitaisreal)). +* fasttest: add timings for "Test time" column in the results table [#35019](https://github.com/ClickHouse/ClickHouse/pull/35019) ([Azat Khuzhin](https://github.com/azat)). +* Always write preprocessed config in XML format [#35020](https://github.com/ClickHouse/ClickHouse/pull/35020) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix read old records from logs [#35021](https://github.com/ClickHouse/ClickHouse/pull/35021) ([alesapin](https://github.com/alesapin)). +* Fix trim function [#35046](https://github.com/ClickHouse/ClickHouse/pull/35046) ([Vladimir C](https://github.com/vdimir)). +* Skip 01086_odbc_roundtrip for aarch, disable force_tests [#35058](https://github.com/ClickHouse/ClickHouse/pull/35058) ([Vladimir C](https://github.com/vdimir)). +* Remove useless define [#35066](https://github.com/ClickHouse/ClickHouse/pull/35066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Take flush_time into account for scheduling background flush of the Buffer [#35071](https://github.com/ClickHouse/ClickHouse/pull/35071) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix failed tests [#35073](https://github.com/ClickHouse/ClickHouse/pull/35073) ([Kruglov Pavel](https://github.com/Avogar)). +* Do not hide exceptions during mutations [#35080](https://github.com/ClickHouse/ClickHouse/pull/35080) ([Azat Khuzhin](https://github.com/azat)). +* Put downloaded artifacts to a temorary path [#35088](https://github.com/ClickHouse/ClickHouse/pull/35088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of duplicate query planing. [#35094](https://github.com/ClickHouse/ClickHouse/pull/35094) ([Amos Bird](https://github.com/amosbird)). +* Temporary supress move partition long for storage S3 [#35097](https://github.com/ClickHouse/ClickHouse/pull/35097) ([alesapin](https://github.com/alesapin)). +* Fix inconsistency in DiskLocal [#35099](https://github.com/ClickHouse/ClickHouse/pull/35099) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try to build llvm for Aarch64 [#35103](https://github.com/ClickHouse/ClickHouse/pull/35103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix build fail with gcc [#35123](https://github.com/ClickHouse/ClickHouse/pull/35123) ([zhanghuajie](https://github.com/zhanghuajieHIT)). +* Fix hardcoded page size [#35129](https://github.com/ClickHouse/ClickHouse/pull/35129) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Delete nodes with attributes `remove` or `replace` if they didn't merge [#35141](https://github.com/ClickHouse/ClickHouse/pull/35141) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Upgrade icu to icu-release-70-1 [#35142](https://github.com/ClickHouse/ClickHouse/pull/35142) ([Yong Wang](https://github.com/kashwy)). +* Merging [#33398](https://github.com/ClickHouse/ClickHouse/issues/33398) [#35157](https://github.com/ClickHouse/ClickHouse/pull/35157) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix typos [#35174](https://github.com/ClickHouse/ClickHouse/pull/35174) ([Zhang Yifan](https://github.com/zhangyifan27)). +* Adjust max_memory_usage in external_aggregation.sql [#35194](https://github.com/ClickHouse/ClickHouse/pull/35194) ([Vladimir C](https://github.com/vdimir)). +* MaterializedMySQL protocol example fix [#35198](https://github.com/ClickHouse/ClickHouse/pull/35198) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove useless files [#35199](https://github.com/ClickHouse/ClickHouse/pull/35199) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless file [#35200](https://github.com/ClickHouse/ClickHouse/pull/35200) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Segment script to of docs pages [#35203](https://github.com/ClickHouse/ClickHouse/pull/35203) ([Rich Raposa](https://github.com/rfraposa)). +* Cache fix [#35209](https://github.com/ClickHouse/ClickHouse/pull/35209) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use C++14 aliases for some type traits [#35212](https://github.com/ClickHouse/ClickHouse/pull/35212) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove redundant configs for TestFlows [#35223](https://github.com/ClickHouse/ClickHouse/pull/35223) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correct some integration tests [#35224](https://github.com/ClickHouse/ClickHouse/pull/35224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone in Docker [#35225](https://github.com/ClickHouse/ClickHouse/pull/35225) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone example in server config [#35226](https://github.com/ClickHouse/ClickHouse/pull/35226) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust timezone in performance tests [#35227](https://github.com/ClickHouse/ClickHouse/pull/35227) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "bugs" that do not exist anymore [#35228](https://github.com/ClickHouse/ClickHouse/pull/35228) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone in stateful tests. [#35229](https://github.com/ClickHouse/ClickHouse/pull/35229) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Moved submodules from ClickHouse-Extras to ClickHouse [#35232](https://github.com/ClickHouse/ClickHouse/pull/35232) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "bugs" that do not exist anymore [#35242](https://github.com/ClickHouse/ClickHouse/pull/35242) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update CachedReadBufferFromRemoteFS.cpp [#35245](https://github.com/ClickHouse/ClickHouse/pull/35245) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#34682](https://github.com/ClickHouse/ClickHouse/issues/34682) [#35247](https://github.com/ClickHouse/ClickHouse/pull/35247) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clang tidy for header files [#35248](https://github.com/ClickHouse/ClickHouse/pull/35248) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible 01506_buffer_table_alter_block_structure_2 flakiness [#35249](https://github.com/ClickHouse/ClickHouse/pull/35249) ([Azat Khuzhin](https://github.com/azat)). +* Mark build action as failed if it was retried and there was no succeeded builds [#35260](https://github.com/ClickHouse/ClickHouse/pull/35260) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test [#35261](https://github.com/ClickHouse/ClickHouse/pull/35261) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix logical error in remote fs cache [#35275](https://github.com/ClickHouse/ClickHouse/pull/35275) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix code style and other minor corrections after implementing allow_no_password. [#35276](https://github.com/ClickHouse/ClickHouse/pull/35276) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add remote fs cache optimization [#35278](https://github.com/ClickHouse/ClickHouse/pull/35278) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove utils/github-hook/hook.py [#35280](https://github.com/ClickHouse/ClickHouse/pull/35280) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix test [#35282](https://github.com/ClickHouse/ClickHouse/pull/35282) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix 00284_external_aggregation.sql [#35293](https://github.com/ClickHouse/ClickHouse/pull/35293) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 00900_long_parquet_load [#35299](https://github.com/ClickHouse/ClickHouse/pull/35299) ([Vladimir C](https://github.com/vdimir)). +* Maybe fix use-after-free inside S3 upload thread [#35305](https://github.com/ClickHouse/ClickHouse/pull/35305) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Modified python packages [#35306](https://github.com/ClickHouse/ClickHouse/pull/35306) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix disappeared host and port from client help message [#35309](https://github.com/ClickHouse/ClickHouse/pull/35309) ([Kruglov Pavel](https://github.com/Avogar)). +* Changelog 22.3 [#35344](https://github.com/ClickHouse/ClickHouse/pull/35344) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace a few uses of enable_if for SFINAE by concepts [#35347](https://github.com/ClickHouse/ClickHouse/pull/35347) ([Robert Schulze](https://github.com/rschu1ze)). + diff --git a/docs/changelogs/v22.3.2.2-lts.md b/docs/changelogs/v22.3.2.2-lts.md index ef45265c7bd..b755db300c8 100644 --- a/docs/changelogs/v22.3.2.2-lts.md +++ b/docs/changelogs/v22.3.2.2-lts.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.2.2-lts FIXME as compared to v22.3.1.1262-prestable diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 4214512c533..4cd48eefa5a 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.3.44-lts FIXME as compared to v22.3.2.2-lts #### Bug Fix @@ -12,3 +19,11 @@ * Backported in [#35856](https://github.com/ClickHouse/ClickHouse/issues/35856): Respect only quota & period from groups, ignore shares (which are not really limit the number of the cores which can be used). [#35815](https://github.com/ClickHouse/ClickHouse/pull/35815) ([filimonov](https://github.com/filimonov)). * Backported in [#35938](https://github.com/ClickHouse/ClickHouse/issues/35938): Avoid processing per-column TTL multiple times. [#35820](https://github.com/ClickHouse/ClickHouse/pull/35820) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Slightly better performance of inserts to `Object` type [#35388](https://github.com/ClickHouse/ClickHouse/pull/35388) ([Anton Popov](https://github.com/CurtizJ)). +* Fix race in data type `Object` [#35409](https://github.com/ClickHouse/ClickHouse/pull/35409) ([Anton Popov](https://github.com/CurtizJ)). +* Fix crash with enabled `optimize_functions_to_subcolumns` [#35512](https://github.com/ClickHouse/ClickHouse/pull/35512) ([Anton Popov](https://github.com/CurtizJ)). +* Fix enable LLVM for JIT compilation in CMake [#35770](https://github.com/ClickHouse/ClickHouse/pull/35770) ([Maksim Kita](https://github.com/kitaisreal)). +* Backport release to 22.3 [#35909](https://github.com/ClickHouse/ClickHouse/pull/35909) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v22.3.4.20-lts.md b/docs/changelogs/v22.3.4.20-lts.md index 4bb4f1bf0f4..d820adbdbec 100644 --- a/docs/changelogs/v22.3.4.20-lts.md +++ b/docs/changelogs/v22.3.4.20-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.4.20-lts FIXME as compared to v22.3.3.44-lts #### Build/Testing/Packaging Improvement diff --git a/docs/changelogs/v22.3.5.5-lts.md b/docs/changelogs/v22.3.5.5-lts.md index d1c42807f41..7deff1be416 100644 --- a/docs/changelogs/v22.3.5.5-lts.md +++ b/docs/changelogs/v22.3.5.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.5.5-lts FIXME as compared to v22.3.4.20-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 16cf390c703..4b4772c611a 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -1,6 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.6.5-lts FIXME as compared to v22.3.5.5-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#36795](https://github.com/ClickHouse/ClickHouse/issues/36795): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md new file mode 100644 index 00000000000..14cb8628f09 --- /dev/null +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -0,0 +1,31 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.3.7.28-lts FIXME as compared to v22.3.6.5-lts + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#37715](https://github.com/ClickHouse/ClickHouse/issues/37715): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#37511](https://github.com/ClickHouse/ClickHouse/issues/37511): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#37843](https://github.com/ClickHouse/ClickHouse/issues/37843): Fix segmentation fault in `show create table` from mysql database when it is configured with named collections. Closes [#37683](https://github.com/ClickHouse/ClickHouse/issues/37683). [#37690](https://github.com/ClickHouse/ClickHouse/pull/37690) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#37940](https://github.com/ClickHouse/ClickHouse/issues/37940): Fix setting cast_ipv4_ipv6_default_on_conversion_error for internal cast function. Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#37761](https://github.com/ClickHouse/ClickHouse/pull/37761) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#37926](https://github.com/ClickHouse/ClickHouse/issues/37926): Fix check asof join key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#37172](https://github.com/ClickHouse/ClickHouse/issues/37172): Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#37457](https://github.com/ClickHouse/ClickHouse/issues/37457): Server might fail to start if it cannot resolve hostname of external ClickHouse dictionary. It's fixed. Fixes [#36451](https://github.com/ClickHouse/ClickHouse/issues/36451). [#36463](https://github.com/ClickHouse/ClickHouse/pull/36463) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#36925](https://github.com/ClickHouse/ClickHouse/issues/36925): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). +* Backported in [#37364](https://github.com/ClickHouse/ClickHouse/issues/37364): Fixed problem with infs in `quantileTDigest`. Fixes [#32107](https://github.com/ClickHouse/ClickHouse/issues/32107). [#37021](https://github.com/ClickHouse/ClickHouse/pull/37021) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)). +* Integration tests [#36866](https://github.com/ClickHouse/ClickHouse/pull/36866) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update protobuf files for kafka and rabbitmq [fix integration tests] [#37884](https://github.com/ClickHouse/ClickHouse/pull/37884) ([Nikita Taranov](https://github.com/nickitat)). +* Try fix `test_grpc_protocol/test.py::test_progress` [#37908](https://github.com/ClickHouse/ClickHouse/pull/37908) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index eb1ed6decd3..c202b0b9331 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.1.2305-prestable FIXME as compared to v22.3.1.1262-prestable #### Backward Incompatible Change @@ -236,3 +243,208 @@ * NO CL ENTRY: 'Revert "clang-tidy report issues with Medium priority"'. [#35941](https://github.com/ClickHouse/ClickHouse/pull/35941) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Fix crash in ParallelReadBuffer"'. [#36210](https://github.com/ClickHouse/ClickHouse/pull/36210) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve backup and restore (experimental) [#33985](https://github.com/ClickHouse/ClickHouse/pull/33985) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not leave any queries after test finish (and add a check into clickhouse-test) [#34924](https://github.com/ClickHouse/ClickHouse/pull/34924) ([Azat Khuzhin](https://github.com/azat)). +* libxml2 package under contrib directory is upgraded to 2.9.13 [#35034](https://github.com/ClickHouse/ClickHouse/pull/35034) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add bugfix validate check [#35124](https://github.com/ClickHouse/ClickHouse/pull/35124) ([Vladimir C](https://github.com/vdimir)). +* curl package upgraded to 7.81.0 [#35130](https://github.com/ClickHouse/ClickHouse/pull/35130) ([Deleted user](https://github.com/ghost)). +* Add test for [#26965](https://github.com/ClickHouse/ClickHouse/issues/26965) [#35159](https://github.com/ClickHouse/ClickHouse/pull/35159) ([palegre-tiny](https://github.com/palegre-tiny)). +* clang-tidy report issues with Medium priority [#35184](https://github.com/ClickHouse/ClickHouse/pull/35184) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add build with GCC [#35204](https://github.com/ClickHouse/ClickHouse/pull/35204) ([Azat Khuzhin](https://github.com/azat)). +* Fix-Clang-Tidy-Errors [#35297](https://github.com/ClickHouse/ClickHouse/pull/35297) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Other fix for 00900_long_parquet_load [#35302](https://github.com/ClickHouse/ClickHouse/pull/35302) ([Vladimir C](https://github.com/vdimir)). +* Add more checks with remoteHostsFilter [#35355](https://github.com/ClickHouse/ClickHouse/pull/35355) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy [#35356](https://github.com/ClickHouse/ClickHouse/pull/35356) ([Anton Popov](https://github.com/CurtizJ)). +* Function h3GetRes0Indexes crash fix [#35358](https://github.com/ClickHouse/ClickHouse/pull/35358) ([Maksim Kita](https://github.com/kitaisreal)). +* Function proporationsZTest formatting fix [#35369](https://github.com/ClickHouse/ClickHouse/pull/35369) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version after release [#35374](https://github.com/ClickHouse/ClickHouse/pull/35374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update SECURITY.md [#35375](https://github.com/ClickHouse/ClickHouse/pull/35375) ([Ivan Blinkov](https://github.com/blinkov)). +* Update version_date.tsv after v22.3.2.2-lts [#35377](https://github.com/ClickHouse/ClickHouse/pull/35377) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Function proportionsZTest fix style check [#35380](https://github.com/ClickHouse/ClickHouse/pull/35380) ([Maksim Kita](https://github.com/kitaisreal)). +* Replace more uses of std::enable_if for SFINAE by concepts [#35383](https://github.com/ClickHouse/ClickHouse/pull/35383) ([Robert Schulze](https://github.com/rschu1ze)). +* Slightly better performance of inserts to `Object` type [#35388](https://github.com/ClickHouse/ClickHouse/pull/35388) ([Anton Popov](https://github.com/CurtizJ)). +* Validate some thoughts over making sets [#35395](https://github.com/ClickHouse/ClickHouse/pull/35395) ([Amos Bird](https://github.com/amosbird)). +* Fix race in data type `Object` [#35409](https://github.com/ClickHouse/ClickHouse/pull/35409) ([Anton Popov](https://github.com/CurtizJ)). +* Rename some variables in keeper [#35431](https://github.com/ClickHouse/ClickHouse/pull/35431) ([alesapin](https://github.com/alesapin)). +* fix 02177_issue_31009_pt2.sql [#35445](https://github.com/ClickHouse/ClickHouse/pull/35445) ([Vladimir C](https://github.com/vdimir)). +* mysqlxx PoolWithFailover style fix [#35462](https://github.com/ClickHouse/ClickHouse/pull/35462) ([Maksim Kita](https://github.com/kitaisreal)). +* Resubmit [#21474](https://github.com/ClickHouse/ClickHouse/issues/21474) [#35467](https://github.com/ClickHouse/ClickHouse/pull/35467) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Explicitly set allow_introspection_functions to 0 [#35470](https://github.com/ClickHouse/ClickHouse/pull/35470) ([Nikita Taranov](https://github.com/nickitat)). +* Merging [#30325](https://github.com/ClickHouse/ClickHouse/issues/30325) [#35478](https://github.com/ClickHouse/ClickHouse/pull/35478) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix show create for information_schema [#35480](https://github.com/ClickHouse/ClickHouse/pull/35480) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add retries in backward compatibility check [#35482](https://github.com/ClickHouse/ClickHouse/pull/35482) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve backward compatibility check and stress tests [#35499](https://github.com/ClickHouse/ClickHouse/pull/35499) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix too strict assertion in DDLWorker [#35503](https://github.com/ClickHouse/ClickHouse/pull/35503) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add more validations in mask operations [#35507](https://github.com/ClickHouse/ClickHouse/pull/35507) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with enabled `optimize_functions_to_subcolumns` [#35512](https://github.com/ClickHouse/ClickHouse/pull/35512) ([Anton Popov](https://github.com/CurtizJ)). +* Don't put red cross if jepsen check couldn't wait for build [#35522](https://github.com/ClickHouse/ClickHouse/pull/35522) ([alesapin](https://github.com/alesapin)). +* Fix ClickHouse name typo in caches.md [#35526](https://github.com/ClickHouse/ClickHouse/pull/35526) ([erikbaan](https://github.com/erikbaan)). +* Simplify strip for new packages [#35533](https://github.com/ClickHouse/ClickHouse/pull/35533) ([alesapin](https://github.com/alesapin)). +* Add workflow dispatch [#35535](https://github.com/ClickHouse/ClickHouse/pull/35535) ([alesapin](https://github.com/alesapin)). +* fix clang tidy warning, add nullptr check [#35540](https://github.com/ClickHouse/ClickHouse/pull/35540) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Slightly better integration tests: test_backup_with_other_granularity test_azure_blob_storage_zero_copy_replication [#35543](https://github.com/ClickHouse/ClickHouse/pull/35543) ([Ilya Yatsishin](https://github.com/qoega)). +* ExternalModelsLoader refactoring [#35546](https://github.com/ClickHouse/ClickHouse/pull/35546) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed style check [#35550](https://github.com/ClickHouse/ClickHouse/pull/35550) ([Maksim Kita](https://github.com/kitaisreal)). +* Cleanup test data [#35555](https://github.com/ClickHouse/ClickHouse/pull/35555) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Revert "Decrease data_type_max_parse_depth a little to avoid stack overflow in coroutines" [#35556](https://github.com/ClickHouse/ClickHouse/pull/35556) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test `test_s3_zero_copy_replication` [#35574](https://github.com/ClickHouse/ClickHouse/pull/35574) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix logging in `test_distributed_respect_user_timeouts` [#35575](https://github.com/ClickHouse/ClickHouse/pull/35575) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Increase fiber stack size a bit in attempt to fix stack overflow in tests with address sanitizer [#35578](https://github.com/ClickHouse/ClickHouse/pull/35578) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't run 01318_long_unsuccessful_mutation_zookeeper test in backward compatibility check [#35580](https://github.com/ClickHouse/ClickHouse/pull/35580) ([Kruglov Pavel](https://github.com/Avogar)). +* Make some tests more stable [#35599](https://github.com/ClickHouse/ClickHouse/pull/35599) ([Kruglov Pavel](https://github.com/Avogar)). +* Check all logs for crashes, logical errors, etc in backward compatibility check [#35613](https://github.com/ClickHouse/ClickHouse/pull/35613) ([Kruglov Pavel](https://github.com/Avogar)). +* Update comment about mismatching checksums [#35621](https://github.com/ClickHouse/ClickHouse/pull/35621) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Push only to the new CI DB [#35622](https://github.com/ClickHouse/ClickHouse/pull/35622) ([alesapin](https://github.com/alesapin)). +* Fix optin.cplusplus.UninitializedObject issue [#35626](https://github.com/ClickHouse/ClickHouse/pull/35626) ([larryluogit](https://github.com/larryluogit)). +* test for partition_by using ignore() [#35636](https://github.com/ClickHouse/ClickHouse/pull/35636) ([Denny Crane](https://github.com/den-crane)). +* test for crash _join_with_nullable_lowcardinality [#35551](https://github.com/ClickHouse/ClickHouse/issues/35551) [#35638](https://github.com/ClickHouse/ClickHouse/pull/35638) ([Denny Crane](https://github.com/den-crane)). +* Use compile-time check for `Exception` messages [#35655](https://github.com/ClickHouse/ClickHouse/pull/35655) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky test `01532_primary_key_without_order_by_zookeeper` [#35657](https://github.com/ClickHouse/ClickHouse/pull/35657) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix multiple flaky tests [#35659](https://github.com/ClickHouse/ClickHouse/pull/35659) ([alesapin](https://github.com/alesapin)). +* Fix flaky test `01091_num_threads` [#35663](https://github.com/ClickHouse/ClickHouse/pull/35663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve black check: show diff in the output [#35665](https://github.com/ClickHouse/ClickHouse/pull/35665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use float devision for avg after optimize_fuse_sum_count_avg [#35666](https://github.com/ClickHouse/ClickHouse/pull/35666) ([Vladimir C](https://github.com/vdimir)). +* Disable random settings in Fast Test [#35669](https://github.com/ClickHouse/ClickHouse/pull/35669) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix randomization of max_read_buffer_size [#35675](https://github.com/ClickHouse/ClickHouse/pull/35675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove outdated links from CI [#35676](https://github.com/ClickHouse/ClickHouse/pull/35676) ([alesapin](https://github.com/alesapin)). +* Fix flaky tests 02149_read_in_order_fixed_prefix and 02177_issue_31009 [#35679](https://github.com/ClickHouse/ClickHouse/pull/35679) ([Kruglov Pavel](https://github.com/Avogar)). +* Rerun failed jobs only for failed workflowa [#35685](https://github.com/ClickHouse/ClickHouse/pull/35685) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flaky 01037_polygon_dicts_correctness_fast [#35688](https://github.com/ClickHouse/ClickHouse/pull/35688) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fail CI checks in case of errors in checks (not failed tests) [#35718](https://github.com/ClickHouse/ClickHouse/pull/35718) ([alesapin](https://github.com/alesapin)). +* Try to fix test_global_overcommit_tracker flakyness [#35719](https://github.com/ClickHouse/ClickHouse/pull/35719) ([Dmitry Novik](https://github.com/novikd)). +* Try to run stateful tests in parallel [#35720](https://github.com/ClickHouse/ClickHouse/pull/35720) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Format `test_allowed_url_from_config/test.py` with black [#35721](https://github.com/ClickHouse/ClickHouse/pull/35721) ([Antonio Andelic](https://github.com/antonio2368)). +* Resurrect automatic labelling [#35722](https://github.com/ClickHouse/ClickHouse/pull/35722) ([alesapin](https://github.com/alesapin)). +* Revert "Fix enable LLVM for JIT compilation in CMake" [#35725](https://github.com/ClickHouse/ClickHouse/pull/35725) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not require writable source directory for generating krb5 error tables [#35734](https://github.com/ClickHouse/ClickHouse/pull/35734) ([Azat Khuzhin](https://github.com/azat)). +* Update ci checks server. [#35737](https://github.com/ClickHouse/ClickHouse/pull/35737) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Tests fixes [#35745](https://github.com/ClickHouse/ClickHouse/pull/35745) ([Azat Khuzhin](https://github.com/azat)). +* Use common IOThreadPool for S3 and URL [#35746](https://github.com/ClickHouse/ClickHouse/pull/35746) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix stylecheck [#35754](https://github.com/ClickHouse/ClickHouse/pull/35754) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* extended a description of clickhouse-client compression parameter [#35758](https://github.com/ClickHouse/ClickHouse/pull/35758) ([Denny Crane](https://github.com/den-crane)). +* Resurrect build hash [#35766](https://github.com/ClickHouse/ClickHouse/pull/35766) ([alesapin](https://github.com/alesapin)). +* Fix 00484_preferred_max_column_in_block_size_bytes [#35768](https://github.com/ClickHouse/ClickHouse/pull/35768) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix enable LLVM for JIT compilation in CMake [#35770](https://github.com/ClickHouse/ClickHouse/pull/35770) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse-keeper: correctly handle closed client connection [#35772](https://github.com/ClickHouse/ClickHouse/pull/35772) ([Azat Khuzhin](https://github.com/azat)). +* ci: replace directory system log tables artifacts with tsv [#35773](https://github.com/ClickHouse/ClickHouse/pull/35773) ([Azat Khuzhin](https://github.com/azat)). +* One more try to resurrect build hash [#35774](https://github.com/ClickHouse/ClickHouse/pull/35774) ([alesapin](https://github.com/alesapin)). +* Refactoring QueryPipeline [#35789](https://github.com/ClickHouse/ClickHouse/pull/35789) ([Amos Bird](https://github.com/amosbird)). +* Delete duplicate code [#35798](https://github.com/ClickHouse/ClickHouse/pull/35798) ([xiedeyantu](https://github.com/xiedeyantu)). +* remove unused variable [#35800](https://github.com/ClickHouse/ClickHouse/pull/35800) ([flynn](https://github.com/ucasfl)). +* Make `SortDescription::column_name` always non-empty [#35805](https://github.com/ClickHouse/ClickHouse/pull/35805) ([Nikita Taranov](https://github.com/nickitat)). +* Fix latest_error referenced before assignment [#35807](https://github.com/ClickHouse/ClickHouse/pull/35807) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to fix some integration tests [#35808](https://github.com/ClickHouse/ClickHouse/pull/35808) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better error message for incorrect *MergeTree arguments [#35814](https://github.com/ClickHouse/ClickHouse/pull/35814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix missing `noexcept(false)` flag on destructor which causes builds to fail [#35817](https://github.com/ClickHouse/ClickHouse/pull/35817) ([tcoyvwac](https://github.com/tcoyvwac)). +* Try remove unneed variable [#35833](https://github.com/ClickHouse/ClickHouse/pull/35833) ([flynn](https://github.com/ucasfl)). +* Refactoring of hints for column descriptor [#35852](https://github.com/ClickHouse/ClickHouse/pull/35852) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix automatic bucket prefix for master [#35854](https://github.com/ClickHouse/ClickHouse/pull/35854) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Format changes for new docs [#35858](https://github.com/ClickHouse/ClickHouse/pull/35858) ([Rich Raposa](https://github.com/rfraposa)). +* Properly cancel the query after client format error [#35867](https://github.com/ClickHouse/ClickHouse/pull/35867) ([Azat Khuzhin](https://github.com/azat)). +* Drop modernize-replace-auto-ptr from .clang-tidy [#35868](https://github.com/ClickHouse/ClickHouse/pull/35868) ([Robert Schulze](https://github.com/rschu1ze)). +* fix service start with systemd [#35869](https://github.com/ClickHouse/ClickHouse/pull/35869) ([Denny Crane](https://github.com/den-crane)). +* fix postgres test [#35885](https://github.com/ClickHouse/ClickHouse/pull/35885) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update poco: Fix polling of socket with negative timeout (when poll() interrupted by EINTR) [#35899](https://github.com/ClickHouse/ClickHouse/pull/35899) ([Azat Khuzhin](https://github.com/azat)). +* More logs on unsuccessful part removal [#35904](https://github.com/ClickHouse/ClickHouse/pull/35904) ([alesapin](https://github.com/alesapin)). +* Executable user defined functions prevent executing during analysis [#35917](https://github.com/ClickHouse/ClickHouse/pull/35917) ([Maksim Kita](https://github.com/kitaisreal)). +* JIT ProfileEvents added test [#35918](https://github.com/ClickHouse/ClickHouse/pull/35918) ([Maksim Kita](https://github.com/kitaisreal)). +* Clang tidy issues [#35919](https://github.com/ClickHouse/ClickHouse/pull/35919) ([Deleted user](https://github.com/ghost)). +* Fix race in cached buffer [#35922](https://github.com/ClickHouse/ClickHouse/pull/35922) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix building ubuntu image from deb-repo [#35931](https://github.com/ClickHouse/ClickHouse/pull/35931) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move credentials for integration tests into a separate file [#35936](https://github.com/ClickHouse/ClickHouse/pull/35936) ([Lalit Srikant](https://github.com/LAL2211)). +* [Snyk] Security upgrade node from 8 to 16.14.2 [#35942](https://github.com/ClickHouse/ClickHouse/pull/35942) ([Snyk bot](https://github.com/snyk-bot)). +* Fix keeper image for `arm64` [#35945](https://github.com/ClickHouse/ClickHouse/pull/35945) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make some replicated DDL queries faster [#35946](https://github.com/ClickHouse/ClickHouse/pull/35946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update Contentsquare company case [#35964](https://github.com/ClickHouse/ClickHouse/pull/35964) ([François Violette](https://github.com/fviolette)). +* Small code changes in ZooKeeper client/Keeper code [#35967](https://github.com/ClickHouse/ClickHouse/pull/35967) ([Antonio Andelic](https://github.com/antonio2368)). +* parallel reading files for FileLog Engine [#35980](https://github.com/ClickHouse/ClickHouse/pull/35980) ([flynn](https://github.com/ucasfl)). +* Fix data race in StorageURL [#35984](https://github.com/ClickHouse/ClickHouse/pull/35984) ([Antonio Andelic](https://github.com/antonio2368)). +* Improve logs analysis in stress test checks [#35985](https://github.com/ClickHouse/ClickHouse/pull/35985) ([Azat Khuzhin](https://github.com/azat)). +* Print labels to log in run_check.py [#35991](https://github.com/ClickHouse/ClickHouse/pull/35991) ([Vladimir C](https://github.com/vdimir)). +* Set `ENABLE_BUILD_PATH_MAPPING` to `OFF` by default, if `CMAKE_BUILD_TYPE` is set to `Debug` [#35998](https://github.com/ClickHouse/ClickHouse/pull/35998) ([Denis Glazachev](https://github.com/traceon)). +* Make test 00159_parallel_formatting_tsv_and_friends.sh more stable [#36001](https://github.com/ClickHouse/ClickHouse/pull/36001) ([Kruglov Pavel](https://github.com/Avogar)). +* Update version_date.tsv after v22.3.3.44-lts [#36002](https://github.com/ClickHouse/ClickHouse/pull/36002) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix flaky test `test_system_merges/test.py::test_merge_simple` [#36004](https://github.com/ClickHouse/ClickHouse/pull/36004) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Another fix for settings randomization [#36005](https://github.com/ClickHouse/ClickHouse/pull/36005) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove some TestFlows that are irrelevant. [#36006](https://github.com/ClickHouse/ClickHouse/pull/36006) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix 02248_nullable_custom_types_to_string [#36008](https://github.com/ClickHouse/ClickHouse/pull/36008) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test 00155_long_merges [#36009](https://github.com/ClickHouse/ClickHouse/pull/36009) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unused code from TestFlows [#36010](https://github.com/ClickHouse/ClickHouse/pull/36010) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix data race in StorgeFileLog [#36015](https://github.com/ClickHouse/ClickHouse/pull/36015) ([flynn](https://github.com/ucasfl)). +* Fix Backward comapatibility check [#36026](https://github.com/ClickHouse/ClickHouse/pull/36026) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use storage_snapshot for projection analysis [#36034](https://github.com/ClickHouse/ClickHouse/pull/36034) ([Amos Bird](https://github.com/amosbird)). +* Nightly coverity [#36044](https://github.com/ClickHouse/ClickHouse/pull/36044) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix typo [#36047](https://github.com/ClickHouse/ClickHouse/pull/36047) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test [#36054](https://github.com/ClickHouse/ClickHouse/pull/36054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trailing whitespace in default client_name [#36056](https://github.com/ClickHouse/ClickHouse/pull/36056) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-client: fix query cancellation if any result was not received yet [#36057](https://github.com/ClickHouse/ClickHouse/pull/36057) ([Azat Khuzhin](https://github.com/azat)). +* Add debug and fix cancel_rerun lambda [#36064](https://github.com/ClickHouse/ClickHouse/pull/36064) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix style in run_check.py [#36065](https://github.com/ClickHouse/ClickHouse/pull/36065) ([Vladimir C](https://github.com/vdimir)). +* Try to turn off always green backward compatibility checks [#36067](https://github.com/ClickHouse/ClickHouse/pull/36067) ([Kruglov Pavel](https://github.com/Avogar)). +* Less dependencies from disks in buffers [#36070](https://github.com/ClickHouse/ClickHouse/pull/36070) ([alesapin](https://github.com/alesapin)). +* Backups: improve arguments handling and file removing [#36072](https://github.com/ClickHouse/ClickHouse/pull/36072) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add tonickkozlov to trusted users for auto ci run [#36080](https://github.com/ClickHouse/ClickHouse/pull/36080) ([nvartolomei](https://github.com/nvartolomei)). +* Try to avoid timeoutes in parallel parsing tests [#36083](https://github.com/ClickHouse/ClickHouse/pull/36083) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix BUILD_NAME issue in build jobs [#36085](https://github.com/ClickHouse/ClickHouse/pull/36085) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove incorrect assertion [#36086](https://github.com/ClickHouse/ClickHouse/pull/36086) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of fs paths keeper [#36106](https://github.com/ClickHouse/ClickHouse/pull/36106) ([alesapin](https://github.com/alesapin)). +* fix typo [#36110](https://github.com/ClickHouse/ClickHouse/pull/36110) ([Denny Crane](https://github.com/den-crane)). +* Fix scan report filename [#36112](https://github.com/ClickHouse/ClickHouse/pull/36112) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add new/delete overloads with align [#36125](https://github.com/ClickHouse/ClickHouse/pull/36125) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix code style in registerBackupEnginesFileAndDisk.cpp [#36127](https://github.com/ClickHouse/ClickHouse/pull/36127) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix flaky tests 00971 and 01003 [#36128](https://github.com/ClickHouse/ClickHouse/pull/36128) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update 01171_mv_select_insert_isolation_long.sh [#36131](https://github.com/ClickHouse/ClickHouse/pull/36131) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build report [#36132](https://github.com/ClickHouse/ClickHouse/pull/36132) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix after [#35475](https://github.com/ClickHouse/ClickHouse/issues/35475) [#36135](https://github.com/ClickHouse/ClickHouse/pull/36135) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor fix for transactions [#36136](https://github.com/ClickHouse/ClickHouse/pull/36136) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ProfileEvents fixes [#36137](https://github.com/ClickHouse/ClickHouse/pull/36137) ([Azat Khuzhin](https://github.com/azat)). +* Minor improvement for hung check [#36138](https://github.com/ClickHouse/ClickHouse/pull/36138) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Upload ccache for the first run in PR [#36139](https://github.com/ClickHouse/ClickHouse/pull/36139) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix test 02241_remote_filesystem_cache_on_insert for database Ordinary [#36150](https://github.com/ClickHouse/ClickHouse/pull/36150) ([Kseniia Sumarokova](https://github.com/kssenii)). +* remove unused array in h3Res0Indexes func [#36154](https://github.com/ClickHouse/ClickHouse/pull/36154) ([Bharat Nallan](https://github.com/bharatnc)). +* fix typo in cmake message [#36155](https://github.com/ClickHouse/ClickHouse/pull/36155) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix UT error sometimes [#36157](https://github.com/ClickHouse/ClickHouse/pull/36157) ([zhanglistar](https://github.com/zhanglistar)). +* Fix a logical error left after debugging [#36159](https://github.com/ClickHouse/ClickHouse/pull/36159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)). +* Some fixes for ReplicatedMergeTree [#36163](https://github.com/ClickHouse/ClickHouse/pull/36163) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-client: properly cancel query in case of error during formatting data [#36164](https://github.com/ClickHouse/ClickHouse/pull/36164) ([Azat Khuzhin](https://github.com/azat)). +* Fix flacky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert "Fix possible mutation stuck due to race with DROP_RANGE" [#36190](https://github.com/ClickHouse/ClickHouse/pull/36190) ([Azat Khuzhin](https://github.com/azat)). +* Use atomic instead of mutex + condvar in ParallelReadBuffer [#36192](https://github.com/ClickHouse/ClickHouse/pull/36192) ([Kruglov Pavel](https://github.com/Avogar)). +* Follow-up to [#36138](https://github.com/ClickHouse/ClickHouse/issues/36138) [#36194](https://github.com/ClickHouse/ClickHouse/pull/36194) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Revert reverting "Fix crash in ParallelReadBuffer" [#36212](https://github.com/ClickHouse/ClickHouse/pull/36212) ([Kruglov Pavel](https://github.com/Avogar)). +* Make stateless tests with s3 always green [#36214](https://github.com/ClickHouse/ClickHouse/pull/36214) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add Tyler Hannan to contributors [#36216](https://github.com/ClickHouse/ClickHouse/pull/36216) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix the repeated call of func to get the table when drop table [#36248](https://github.com/ClickHouse/ClickHouse/pull/36248) ([xiedeyantu](https://github.com/xiedeyantu)). +* Split test 01675_data_type_coroutine into 2 tests to prevent possible timeouts [#36250](https://github.com/ClickHouse/ClickHouse/pull/36250) ([Kruglov Pavel](https://github.com/Avogar)). +* Merge TRUSTED_CONTRIBUTORS in lambda and import in check [#36252](https://github.com/ClickHouse/ClickHouse/pull/36252) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix exception "File segment can be completed only by downloader" in tests [#36253](https://github.com/ClickHouse/ClickHouse/pull/36253) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix integration tests report parser [#36257](https://github.com/ClickHouse/ClickHouse/pull/36257) ([alesapin](https://github.com/alesapin)). +* fix crash when you use clickhouse-git-import with invalid parameter [#36262](https://github.com/ClickHouse/ClickHouse/pull/36262) ([zhanghuajie](https://github.com/zhanghuajieHIT)). +* Fix cancel-lambda for closed PRs [#36269](https://github.com/ClickHouse/ClickHouse/pull/36269) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Place new logic of schema inference in insert select from table function under setting [#36275](https://github.com/ClickHouse/ClickHouse/pull/36275) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove trash from CPUID [#36310](https://github.com/ClickHouse/ClickHouse/pull/36310) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove remaining parts of Arcadia [#36312](https://github.com/ClickHouse/ClickHouse/pull/36312) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang-tidy-14 (part 1) [#36320](https://github.com/ClickHouse/ClickHouse/pull/36320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add dispatch trigger for debug CI [#36329](https://github.com/ClickHouse/ClickHouse/pull/36329) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Revert "support UNSIGNED modifier with unused parameters of INT" [#36337](https://github.com/ClickHouse/ClickHouse/pull/36337) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Simplify perf scripts by using schema inference [#36346](https://github.com/ClickHouse/ClickHouse/pull/36346) ([Azat Khuzhin](https://github.com/azat)). +* Do not randomize "priority" setting [#36358](https://github.com/ClickHouse/ClickHouse/pull/36358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration tests [#36361](https://github.com/ClickHouse/ClickHouse/pull/36361) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove abbreviation [#36362](https://github.com/ClickHouse/ClickHouse/pull/36362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix slow test [#36363](https://github.com/ClickHouse/ClickHouse/pull/36363) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error in sanity checks [#36365](https://github.com/ClickHouse/ClickHouse/pull/36365) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add nodiscard attribute to immutable methods of `IColumn` to avoid errors [#36368](https://github.com/ClickHouse/ClickHouse/pull/36368) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix strange trash in Keeper [#36369](https://github.com/ClickHouse/ClickHouse/pull/36369) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix typo [#36374](https://github.com/ClickHouse/ClickHouse/pull/36374) ([flynn](https://github.com/ucasfl)). +* Fix test 01161_all_system_tables under s3 storage [#36388](https://github.com/ClickHouse/ClickHouse/pull/36388) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor refactor to prefer C++ Standard Algorithms [#36393](https://github.com/ClickHouse/ClickHouse/pull/36393) ([tcoyvwac](https://github.com/tcoyvwac)). +* jemalloc: enable logging in debug build [#36398](https://github.com/ClickHouse/ClickHouse/pull/36398) ([Azat Khuzhin](https://github.com/azat)). +* Respect library type for contrib libraries [#36399](https://github.com/ClickHouse/ClickHouse/pull/36399) ([Azat Khuzhin](https://github.com/azat)). +* Add more harmful variables for OSX [#36400](https://github.com/ClickHouse/ClickHouse/pull/36400) ([Azat Khuzhin](https://github.com/azat)). +* Activate clang-tidy warning "readability-container-contains" [#36402](https://github.com/ClickHouse/ClickHouse/pull/36402) ([Robert Schulze](https://github.com/rschu1ze)). +* Move down data_range construction in filterMarksUsingIndex and filterMarksUsingMergedIndex [#36410](https://github.com/ClickHouse/ClickHouse/pull/36410) ([Tian Xinhui](https://github.com/xinhuitian)). +* Fix stress test [#36450](https://github.com/ClickHouse/ClickHouse/pull/36450) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v22.4.2.1-stable.md b/docs/changelogs/v22.4.2.1-stable.md index cd7ee75997c..fb77d3fee9b 100644 --- a/docs/changelogs/v22.4.2.1-stable.md +++ b/docs/changelogs/v22.4.2.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.2.1-stable FIXME as compared to v22.4.1.2305-prestable diff --git a/docs/changelogs/v22.4.3.3-stable.md b/docs/changelogs/v22.4.3.3-stable.md index 5ab7872f880..4baa63672ab 100644 --- a/docs/changelogs/v22.4.3.3-stable.md +++ b/docs/changelogs/v22.4.3.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.3.3-stable FIXME as compared to v22.4.2.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 9004ce2f6bc..71e077ac071 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.4.7-stable FIXME as compared to v22.4.3.3-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -5,3 +12,7 @@ * Backported in [#36524](https://github.com/ClickHouse/ClickHouse/issues/36524): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36673](https://github.com/ClickHouse/ClickHouse/issues/36673): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index ab43bf3eade..636ad2ed3ac 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.5.9-stable FIXME as compared to v22.4.4.7-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,7 @@ * Backported in [#36794](https://github.com/ClickHouse/ClickHouse/issues/36794): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#36926](https://github.com/ClickHouse/ClickHouse/issues/36926): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Some fixes for replicated merge tree [#36909](https://github.com/ClickHouse/ClickHouse/pull/36909) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index aab8266c115..dfdcad64561 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.5.1.2079-stable FIXME as compared to v22.4.1.2305-prestable #### Backward Incompatible Change @@ -180,3 +187,204 @@ * NO CL ENTRY: 'Revert "BLAKE3 hash function documentation"'. [#37092](https://github.com/ClickHouse/ClickHouse/pull/37092) ([Rich Raposa](https://github.com/rfraposa)). * NO CL ENTRY: 'Revert "Remove height restrictions from the query div in play web tool."'. [#37261](https://github.com/ClickHouse/ClickHouse/pull/37261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Cleanup GROUPING SETS formating [#32999](https://github.com/ClickHouse/ClickHouse/pull/32999) ([Dmitry Novik](https://github.com/novikd)). +* Minor renames [#35272](https://github.com/ClickHouse/ClickHouse/pull/35272) ([Anton Popov](https://github.com/CurtizJ)). +* clickhouse-test: fix left-queries-check, to fix test log parser [#35865](https://github.com/ClickHouse/ClickHouse/pull/35865) ([Azat Khuzhin](https://github.com/azat)). +* Regression test for CHECKSUM_DOESNT_MATCH error because of per-column TTL bug [#35971](https://github.com/ClickHouse/ClickHouse/pull/35971) ([Azat Khuzhin](https://github.com/azat)). +* Fix performance tests [#35976](https://github.com/ClickHouse/ClickHouse/pull/35976) ([Maksim Kita](https://github.com/kitaisreal)). +* Backup for replicated tables and other improvements [#36198](https://github.com/ClickHouse/ClickHouse/pull/36198) ([Vitaly Baranov](https://github.com/vitlibar)). +* Trying to fix some trash in zero copy replication [#36299](https://github.com/ClickHouse/ClickHouse/pull/36299) ([alesapin](https://github.com/alesapin)). +* Speed up build a little [#36319](https://github.com/ClickHouse/ClickHouse/pull/36319) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace remove-erase idiom by C++20 erase()/erase_if() [#36348](https://github.com/ClickHouse/ClickHouse/pull/36348) ([Robert Schulze](https://github.com/rschu1ze)). +* Strict taskstats parser [#36351](https://github.com/ClickHouse/ClickHouse/pull/36351) ([Azat Khuzhin](https://github.com/azat)). +* Draft changelog for version 22.4 [#36397](https://github.com/ClickHouse/ClickHouse/pull/36397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update int-uint.md [#36404](https://github.com/ClickHouse/ClickHouse/pull/36404) ([hardstep33](https://github.com/hardstep33)). +* Return back [#36126](https://github.com/ClickHouse/ClickHouse/issues/36126) [#36423](https://github.com/ClickHouse/ClickHouse/pull/36423) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed warnings of clang-tidy check "bugprone-branch-clone" [#36431](https://github.com/ClickHouse/ClickHouse/pull/36431) ([Robert Schulze](https://github.com/rschu1ze)). +* Clang tidy fixes [#36444](https://github.com/ClickHouse/ClickHouse/pull/36444) ([Robert Schulze](https://github.com/rschu1ze)). +* fixed /common/example cow_compositions.cpp clone the inner column when there is no need [#36453](https://github.com/ClickHouse/ClickHouse/pull/36453) ([zombee0](https://github.com/zombee0)). +* Refactoring dependency for ParserAttachAccessEntity [#36468](https://github.com/ClickHouse/ClickHouse/pull/36468) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* more specific warning text about low disk space [#36472](https://github.com/ClickHouse/ClickHouse/pull/36472) ([Sergei Trifonov](https://github.com/serxa)). +* Fixed missing enum values for ClientInfo::Interface [#36482](https://github.com/ClickHouse/ClickHouse/pull/36482) ([Vasily Nemkov](https://github.com/Enmk)). +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Update version after release [#36502](https://github.com/ClickHouse/ClickHouse/pull/36502) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Followup on [#36172](https://github.com/ClickHouse/ClickHouse/issues/36172) password hash salt feature [#36510](https://github.com/ClickHouse/ClickHouse/pull/36510) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update version_date.tsv after v22.4.2.1-stable [#36533](https://github.com/ClickHouse/ClickHouse/pull/36533) ([github-actions[bot]](https://github.com/apps/github-actions)). +* fix log should print 'from' path [#36535](https://github.com/ClickHouse/ClickHouse/pull/36535) ([xiedeyantu](https://github.com/xiedeyantu)). +* Add function bin tests for Int/UInt128/UInt256 [#36537](https://github.com/ClickHouse/ClickHouse/pull/36537) ([Memo](https://github.com/Joeywzr)). +* Fix 01161_all_system_tables [#36539](https://github.com/ClickHouse/ClickHouse/pull/36539) ([Antonio Andelic](https://github.com/antonio2368)). +* Update PULL_REQUEST_TEMPLATE.md [#36543](https://github.com/ClickHouse/ClickHouse/pull/36543) ([Ivan Blinkov](https://github.com/blinkov)). +* Fixed integer overflow in toStartOfInterval [#36546](https://github.com/ClickHouse/ClickHouse/pull/36546) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix issue with broken git ownership [#36548](https://github.com/ClickHouse/ClickHouse/pull/36548) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix version autodetection for docker_server.py [#36552](https://github.com/ClickHouse/ClickHouse/pull/36552) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Snyk fixes [#36554](https://github.com/ClickHouse/ClickHouse/pull/36554) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use just index to split performance tests by group [#36559](https://github.com/ClickHouse/ClickHouse/pull/36559) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update quantiledeterministic.md [#36560](https://github.com/ClickHouse/ClickHouse/pull/36560) ([ifinik](https://github.com/ifinik)). +* CodeQL Experiment - Exclude contrib Dependencies [#36561](https://github.com/ClickHouse/ClickHouse/pull/36561) ([Julio Jimenez](https://github.com/juliojimenez)). +* Small refactoring of Processors and QueryPipeline [#36579](https://github.com/ClickHouse/ClickHouse/pull/36579) ([Amos Bird](https://github.com/amosbird)). +* Simplify 01834_alias_columns_laziness_filimonov test [#36585](https://github.com/ClickHouse/ClickHouse/pull/36585) ([Azat Khuzhin](https://github.com/azat)). +* bash-completion: add completion for send_logs_level [#36586](https://github.com/ClickHouse/ClickHouse/pull/36586) ([Azat Khuzhin](https://github.com/azat)). +* client: add a message on reconnect (under warning log level) [#36587](https://github.com/ClickHouse/ClickHouse/pull/36587) ([Azat Khuzhin](https://github.com/azat)). +* Fix strange whitespace (or I do not know YAML) [#36590](https://github.com/ClickHouse/ClickHouse/pull/36590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Strict taskstats parser (resubmit) [#36602](https://github.com/ClickHouse/ClickHouse/pull/36602) ([Azat Khuzhin](https://github.com/azat)). +* Exclude test `02271_fix_column_matcher_and_column_transformer` from bc check [#36607](https://github.com/ClickHouse/ClickHouse/pull/36607) ([alesapin](https://github.com/alesapin)). +* Ancient cmake version cleanup [#36612](https://github.com/ClickHouse/ClickHouse/pull/36612) ([Robert Schulze](https://github.com/rschu1ze)). +* Cleanup clang-tidy integration. [#36613](https://github.com/ClickHouse/ClickHouse/pull/36613) ([Robert Schulze](https://github.com/rschu1ze)). +* ParallelReadBuffer small improvements [#36619](https://github.com/ClickHouse/ClickHouse/pull/36619) ([Antonio Andelic](https://github.com/antonio2368)). +* Lambda cancel sync [#36622](https://github.com/ClickHouse/ClickHouse/pull/36622) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Version history benchmarks [#36628](https://github.com/ClickHouse/ClickHouse/pull/36628) ([Ilya Yatsishin](https://github.com/qoega)). +* fix typo [#36629](https://github.com/ClickHouse/ClickHouse/pull/36629) ([Sergei Trifonov](https://github.com/serxa)). +* Tiny Mutator code cleanup [#36630](https://github.com/ClickHouse/ClickHouse/pull/36630) ([Azat Khuzhin](https://github.com/azat)). +* Disble test postgresql replica with asan [#36631](https://github.com/ClickHouse/ClickHouse/pull/36631) ([alesapin](https://github.com/alesapin)). +* Minor Coverity defects fixes [#36632](https://github.com/ClickHouse/ClickHouse/pull/36632) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Properly wait for queries in 01502_long_log_tinylog_deadlock_race test [#36634](https://github.com/ClickHouse/ClickHouse/pull/36634) ([Azat Khuzhin](https://github.com/azat)). +* remove unneeded if statement [#36636](https://github.com/ClickHouse/ClickHouse/pull/36636) ([flynn](https://github.com/ucasfl)). +* Fix SortingStep::updateOutputStream() [#36638](https://github.com/ClickHouse/ClickHouse/pull/36638) ([Nikita Taranov](https://github.com/nickitat)). +* CodeQL - Run Daily Analysis [#36640](https://github.com/ClickHouse/ClickHouse/pull/36640) ([Julio Jimenez](https://github.com/juliojimenez)). +* Simplify check_branch, prefetch target branches [#36641](https://github.com/ClickHouse/ClickHouse/pull/36641) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Make backward compatible integration tests runner [#36643](https://github.com/ClickHouse/ClickHouse/pull/36643) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix some flaky tests [#36644](https://github.com/ClickHouse/ClickHouse/pull/36644) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update version_date.tsv after v22.4.3.3-stable [#36651](https://github.com/ClickHouse/ClickHouse/pull/36651) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix flaky test [#36652](https://github.com/ClickHouse/ClickHouse/pull/36652) ([Amos Bird](https://github.com/amosbird)). +* add missing pandas package [#36653](https://github.com/ClickHouse/ClickHouse/pull/36653) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Fix style issue reported by black formatter [#36655](https://github.com/ClickHouse/ClickHouse/pull/36655) ([Alexander Gololobov](https://github.com/davenger)). +* Fix formatting in drop cache system query [#36658](https://github.com/ClickHouse/ClickHouse/pull/36658) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix stress test after 36639 [#36660](https://github.com/ClickHouse/ClickHouse/pull/36660) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update version_date.tsv after v22.3.4.20-lts [#36668](https://github.com/ClickHouse/ClickHouse/pull/36668) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Update 00170_s3_cache.sql [#36669](https://github.com/ClickHouse/ClickHouse/pull/36669) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better version of SeekableReadBufferWithSize [#36676](https://github.com/ClickHouse/ClickHouse/pull/36676) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better logs for virtual parts [#36680](https://github.com/ClickHouse/ClickHouse/pull/36680) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Check socket is connected in HTTPSession [#36683](https://github.com/ClickHouse/ClickHouse/pull/36683) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix typo [#36684](https://github.com/ClickHouse/ClickHouse/pull/36684) ([flynn](https://github.com/ucasfl)). +* Remove excessive logging from S3 [#36689](https://github.com/ClickHouse/ClickHouse/pull/36689) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tests report [#36701](https://github.com/ClickHouse/ClickHouse/pull/36701) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use correct nodes in test_keeper_znode_time [#36711](https://github.com/ClickHouse/ClickHouse/pull/36711) ([Antonio Andelic](https://github.com/antonio2368)). +* Reorganize source files so that base won't depend on Common [#36715](https://github.com/ClickHouse/ClickHouse/pull/36715) ([Amos Bird](https://github.com/amosbird)). +* tests: fix 02015_async_inserts_stress_long flakiness [#36731](https://github.com/ClickHouse/ClickHouse/pull/36731) ([Azat Khuzhin](https://github.com/azat)). +* Add an extra check for RAFT config change [#36736](https://github.com/ClickHouse/ClickHouse/pull/36736) ([Antonio Andelic](https://github.com/antonio2368)). +* Update 00170_s3_cache.sql [#36743](https://github.com/ClickHouse/ClickHouse/pull/36743) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ClickHouseDictionarySource context copy [#36744](https://github.com/ClickHouse/ClickHouse/pull/36744) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix build SeekableReadBufferWithSize -> SeekableReadBuffer [#36745](https://github.com/ClickHouse/ClickHouse/pull/36745) ([Vladimir C](https://github.com/vdimir)). +* Better error message from NuRaft in case of CRC mismatch [#36746](https://github.com/ClickHouse/ClickHouse/pull/36746) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix certs (finishing pr [#36457](https://github.com/ClickHouse/ClickHouse/issues/36457)) [#36747](https://github.com/ClickHouse/ClickHouse/pull/36747) ([Nikita Taranov](https://github.com/nickitat)). +* Another fix for Hung Check [#36752](https://github.com/ClickHouse/ClickHouse/pull/36752) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Deduce `UInt8` type for bools from json instead of `UInt64` [#36756](https://github.com/ClickHouse/ClickHouse/pull/36756) ([Anton Popov](https://github.com/CurtizJ)). +* Add small script for keeper check [#36758](https://github.com/ClickHouse/ClickHouse/pull/36758) ([alesapin](https://github.com/alesapin)). +* Fix ungrammatical error message [#36760](https://github.com/ClickHouse/ClickHouse/pull/36760) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test `test_ddl_worker_non_leader` [#36765](https://github.com/ClickHouse/ClickHouse/pull/36765) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix evaluateConstantExpression for subqueries [#36766](https://github.com/ClickHouse/ClickHouse/pull/36766) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix clickhouse-test for server without `is_all_data_sent` in system.processes [#36767](https://github.com/ClickHouse/ClickHouse/pull/36767) ([Azat Khuzhin](https://github.com/azat)). +* Init thread pools for clickhouse-local [#36778](https://github.com/ClickHouse/ClickHouse/pull/36778) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix performance test [#36779](https://github.com/ClickHouse/ClickHouse/pull/36779) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better exception messages while socket timeouts [#36781](https://github.com/ClickHouse/ClickHouse/pull/36781) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve docs_check to be used in clickhouse-docs [#36796](https://github.com/ClickHouse/ClickHouse/pull/36796) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* pull poco #58 [#36798](https://github.com/ClickHouse/ClickHouse/pull/36798) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Tiny cleanup [#36799](https://github.com/ClickHouse/ClickHouse/pull/36799) ([Azat Khuzhin](https://github.com/azat)). +* Use mutex per worker in ParallelReadBuffer [#36801](https://github.com/ClickHouse/ClickHouse/pull/36801) ([Antonio Andelic](https://github.com/antonio2368)). +* Update version_date.tsv after v22.3.5.5-lts [#36805](https://github.com/ClickHouse/ClickHouse/pull/36805) ([github-actions[bot]](https://github.com/apps/github-actions)). +* play.html: add button to transpose table [#36811](https://github.com/ClickHouse/ClickHouse/pull/36811) ([Vladimir C](https://github.com/vdimir)). +* Fix style error in DocsCheck [#36813](https://github.com/ClickHouse/ClickHouse/pull/36813) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Set is_all_data_sent on exceptions too [#36816](https://github.com/ClickHouse/ClickHouse/pull/36816) ([Azat Khuzhin](https://github.com/azat)). +* Clang -Tidy Fixes [#36817](https://github.com/ClickHouse/ClickHouse/pull/36817) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Protection from incorrect build [#36819](https://github.com/ClickHouse/ClickHouse/pull/36819) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid recreation of system.asynchronous_metric_log (due to difference in codec) [#36820](https://github.com/ClickHouse/ClickHouse/pull/36820) ([Azat Khuzhin](https://github.com/azat)). +* Removed forceful drop cache command, fix detached status state [#36825](https://github.com/ClickHouse/ClickHouse/pull/36825) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log hash table's cache messages with TRACE level [#36830](https://github.com/ClickHouse/ClickHouse/pull/36830) ([Nikita Taranov](https://github.com/nickitat)). +* Cleanup: Remove switches for obsolete GCC version [#36831](https://github.com/ClickHouse/ClickHouse/pull/36831) ([Robert Schulze](https://github.com/rschu1ze)). +* Add functions from CREATE FUNCTION to completion [#36834](https://github.com/ClickHouse/ClickHouse/pull/36834) ([Azat Khuzhin](https://github.com/azat)). +* Validate that function had been passed in CREATE FUNCTION [#36835](https://github.com/ClickHouse/ClickHouse/pull/36835) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unimplemented RAID1 support [#36836](https://github.com/ClickHouse/ClickHouse/pull/36836) ([Azat Khuzhin](https://github.com/azat)). +* Disable `merge_tree_metadata_cache` by default [#36838](https://github.com/ClickHouse/ClickHouse/pull/36838) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace uses of SFINAE by C++20 concepts [#36839](https://github.com/ClickHouse/ClickHouse/pull/36839) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix performance test (4) [#36840](https://github.com/ClickHouse/ClickHouse/pull/36840) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "preconditions" from performance tests (overengineering, unneeded feature) [#36841](https://github.com/ClickHouse/ClickHouse/pull/36841) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove inherited create() method + disallow copying of storages [#36844](https://github.com/ClickHouse/ClickHouse/pull/36844) ([Robert Schulze](https://github.com/rschu1ze)). +* tests: disable 02260_alter_compact_part_drop_nested_column for <=22.4 [#36845](https://github.com/ClickHouse/ClickHouse/pull/36845) ([Azat Khuzhin](https://github.com/azat)). +* Fix check black [#36850](https://github.com/ClickHouse/ClickHouse/pull/36850) ([Kruglov Pavel](https://github.com/Avogar)). +* Aggregator JIT compilation lock fix [#36852](https://github.com/ClickHouse/ClickHouse/pull/36852) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Other Query Time Microseconds Profile Event [#36853](https://github.com/ClickHouse/ClickHouse/pull/36853) ([Ilya Yatsishin](https://github.com/qoega)). +* Reproduce and a little bit better fix for LC dict right offset. [#36856](https://github.com/ClickHouse/ClickHouse/pull/36856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Terminate if S3 buffer is not finalized [#36857](https://github.com/ClickHouse/ClickHouse/pull/36857) ([alesapin](https://github.com/alesapin)). +* Activated a bunch of LLVM 12/13/14 clang-tidy warnings [#36862](https://github.com/ClickHouse/ClickHouse/pull/36862) ([Robert Schulze](https://github.com/rschu1ze)). +* Replace `Timeout` with `Tests not finished` [#36863](https://github.com/ClickHouse/ClickHouse/pull/36863) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backup improvements [#36864](https://github.com/ClickHouse/ClickHouse/pull/36864) ([Vitaly Baranov](https://github.com/vitlibar)). +* Integration tests [#36866](https://github.com/ClickHouse/ClickHouse/pull/36866) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Relax cmake check for CFLAGS/CXXFLAGS/LDFLAGS [#36869](https://github.com/ClickHouse/ClickHouse/pull/36869) ([Azat Khuzhin](https://github.com/azat)). +* Fix insertion of complex JSONs with nested arrays [2] [#36873](https://github.com/ClickHouse/ClickHouse/pull/36873) ([Anton Popov](https://github.com/CurtizJ)). +* Add column to system.filesystem_cache_log [#36874](https://github.com/ClickHouse/ClickHouse/pull/36874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log into filesystem_cache_log when cache not even attempted [#36876](https://github.com/ClickHouse/ClickHouse/pull/36876) ([Kseniia Sumarokova](https://github.com/kssenii)). +* fix typo in comment [#36880](https://github.com/ClickHouse/ClickHouse/pull/36880) ([Sergei Trifonov](https://github.com/serxa)). +* Add some CurrentMetrics for fs cache [#36882](https://github.com/ClickHouse/ClickHouse/pull/36882) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add some asynchronous metrics for fs cache [#36883](https://github.com/ClickHouse/ClickHouse/pull/36883) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Save system logs in functionsl tests if server crashed [#36885](https://github.com/ClickHouse/ClickHouse/pull/36885) ([Alexander Tokmakov](https://github.com/tavplubix)). +* quick tmp fix for stress test [#36900](https://github.com/ClickHouse/ClickHouse/pull/36900) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Some fixes for replicated merge tree [#36909](https://github.com/ClickHouse/ClickHouse/pull/36909) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Invert .clang-tidy [#36911](https://github.com/ClickHouse/ClickHouse/pull/36911) ([Robert Schulze](https://github.com/rschu1ze)). +* Replace make_pair()/make_tuple() by pair()/tuple() [#36913](https://github.com/ClickHouse/ClickHouse/pull/36913) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove log message on client reconnects (reverts [#36587](https://github.com/ClickHouse/ClickHouse/issues/36587)) [#36915](https://github.com/ClickHouse/ClickHouse/pull/36915) ([Azat Khuzhin](https://github.com/azat)). +* Fix profile events in fs cached buffer [#36916](https://github.com/ClickHouse/ClickHouse/pull/36916) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Benchmark was not loaded properly [#36918](https://github.com/ClickHouse/ClickHouse/pull/36918) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Force keeper snapshot equality on different replicas [#36947](https://github.com/ClickHouse/ClickHouse/pull/36947) ([alesapin](https://github.com/alesapin)). +* Remove strange code [#36951](https://github.com/ClickHouse/ClickHouse/pull/36951) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nested exception [#36952](https://github.com/ClickHouse/ClickHouse/pull/36952) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clangtidies [#36960](https://github.com/ClickHouse/ClickHouse/pull/36960) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve stress tests report a little bit [#36961](https://github.com/ClickHouse/ClickHouse/pull/36961) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Follow up for ConcurrentHashJoin [#36970](https://github.com/ClickHouse/ClickHouse/pull/36970) ([Vladimir C](https://github.com/vdimir)). +* Fix formats docs [#36972](https://github.com/ClickHouse/ClickHouse/pull/36972) ([Kruglov Pavel](https://github.com/Avogar)). +* Support secure connection in clickhouse-test [#36977](https://github.com/ClickHouse/ClickHouse/pull/36977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix installation_id filter, minimize docker images diff [#36978](https://github.com/ClickHouse/ClickHouse/pull/36978) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update version_date.tsv after v22.3.6.5-lts [#36985](https://github.com/ClickHouse/ClickHouse/pull/36985) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Update version_date.tsv after v22.4.5.9-stable [#36986](https://github.com/ClickHouse/ClickHouse/pull/36986) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Removing ReplacingWindowColumnTransform in WindowView [#36998](https://github.com/ClickHouse/ClickHouse/pull/36998) ([vxider](https://github.com/Vxider)). +* Merging [#34765](https://github.com/ClickHouse/ClickHouse/issues/34765) [#37002](https://github.com/ClickHouse/ClickHouse/pull/37002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove CodeQL [#37006](https://github.com/ClickHouse/ClickHouse/pull/37006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete code [#37009](https://github.com/ClickHouse/ClickHouse/pull/37009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fix flaky test [#37010](https://github.com/ClickHouse/ClickHouse/pull/37010) ([Nikita Taranov](https://github.com/nickitat)). +* fix output error in LabelsCheck [#37016](https://github.com/ClickHouse/ClickHouse/pull/37016) ([wuxiaobai24](https://github.com/wuxiaobai24)). +* remove useless code [#37020](https://github.com/ClickHouse/ClickHouse/pull/37020) ([flynn](https://github.com/ucasfl)). +* Merging [#34932](https://github.com/ClickHouse/ClickHouse/issues/34932). [#37023](https://github.com/ClickHouse/ClickHouse/pull/37023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix benchmark build code [#37025](https://github.com/ClickHouse/ClickHouse/pull/37025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add benchmark script for testing clouds [#37027](https://github.com/ClickHouse/ClickHouse/pull/37027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration mongodb test [#37035](https://github.com/ClickHouse/ClickHouse/pull/37035) ([Kruglov Pavel](https://github.com/Avogar)). +* Print stacks if we cannot terminate server in stress tests [#37052](https://github.com/ClickHouse/ClickHouse/pull/37052) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove last mentions of data streams [#37053](https://github.com/ClickHouse/ClickHouse/pull/37053) ([Anton Popov](https://github.com/CurtizJ)). +* Changelog script [#37057](https://github.com/ClickHouse/ClickHouse/pull/37057) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* This evening I started using Grammarly. [#37058](https://github.com/ClickHouse/ClickHouse/pull/37058) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update sanity checks warning message text [#37063](https://github.com/ClickHouse/ClickHouse/pull/37063) ([Sergei Trifonov](https://github.com/serxa)). +* Continue fixing [#36199](https://github.com/ClickHouse/ClickHouse/issues/36199). [#37071](https://github.com/ClickHouse/ClickHouse/pull/37071) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* just fix a number [#37072](https://github.com/ClickHouse/ClickHouse/pull/37072) ([jiahui-97](https://github.com/jiahui-97)). +* Disable thread fuzzer after server restart [#37079](https://github.com/ClickHouse/ClickHouse/pull/37079) ([alesapin](https://github.com/alesapin)). +* Option to force cross_to_inner_join_rewrite [#37085](https://github.com/ClickHouse/ClickHouse/pull/37085) ([Vladimir C](https://github.com/vdimir)). +* fix wrong argument in proxy resolver of DiskS3 [#37100](https://github.com/ClickHouse/ClickHouse/pull/37100) ([flynn](https://github.com/ucasfl)). +* tests: fix 01119_optimize_trivial_insert_select (due to max_threads randomization) [#37101](https://github.com/ClickHouse/ClickHouse/pull/37101) ([Azat Khuzhin](https://github.com/azat)). +* Support SELECT query in WindowView [#37105](https://github.com/ClickHouse/ClickHouse/pull/37105) ([vxider](https://github.com/Vxider)). +* Fix workflow style check [#37113](https://github.com/ClickHouse/ClickHouse/pull/37113) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add check for columns size in Block::cloneWithColumns [#37124](https://github.com/ClickHouse/ClickHouse/pull/37124) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update documentation and defaults for memory overcommit [#37129](https://github.com/ClickHouse/ClickHouse/pull/37129) ([Dmitry Novik](https://github.com/novikd)). +* Update default remote fs read method in ReadSettings [#37130](https://github.com/ClickHouse/ClickHouse/pull/37130) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update Exception Message for allowed auth types [#37132](https://github.com/ClickHouse/ClickHouse/pull/37132) ([Marcelo Rodriguez](https://github.com/e-mars)). +* tests/integration: fix possible race for iptables user rules inside containers [#37138](https://github.com/ClickHouse/ClickHouse/pull/37138) ([Azat Khuzhin](https://github.com/azat)). +* Fix fasttest ccache permissions [#37143](https://github.com/ClickHouse/ClickHouse/pull/37143) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add const qualifier for few methods from Context [#37154](https://github.com/ClickHouse/ClickHouse/pull/37154) ([Azat Khuzhin](https://github.com/azat)). +* Reload listen_reuse_port/listen_backlog from config [#37156](https://github.com/ClickHouse/ClickHouse/pull/37156) ([Azat Khuzhin](https://github.com/azat)). +* Enable DNS cache for HTTPSClientSession. [#37157](https://github.com/ClickHouse/ClickHouse/pull/37157) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix async reads from remote fs internal setting not being always turned on [#37164](https://github.com/ClickHouse/ClickHouse/pull/37164) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backups Improvements 5 [#37168](https://github.com/ClickHouse/ClickHouse/pull/37168) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add S3Requests metric [#37200](https://github.com/ClickHouse/ClickHouse/pull/37200) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix coverity build problem with LD_PRELOAD [#37203](https://github.com/ClickHouse/ClickHouse/pull/37203) ([Boris Kuschel](https://github.com/bkuschel)). +* Relax log level for some checks in check thread [#37208](https://github.com/ClickHouse/ClickHouse/pull/37208) ([alesapin](https://github.com/alesapin)). +* update poco [#37209](https://github.com/ClickHouse/ClickHouse/pull/37209) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Ignore harmful env variables in clickhouse binaries (reexec w/o them) [#37211](https://github.com/ClickHouse/ClickHouse/pull/37211) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong comment in IVolume.h [#37218](https://github.com/ClickHouse/ClickHouse/pull/37218) ([Sergei Trifonov](https://github.com/serxa)). +* Cmake cleanup pt2 [#37222](https://github.com/ClickHouse/ClickHouse/pull/37222) ([Robert Schulze](https://github.com/rschu1ze)). +* Temporarily fix flaky test `01825_type_json_insert_select.sql` [#37245](https://github.com/ClickHouse/ClickHouse/pull/37245) ([Anton Popov](https://github.com/CurtizJ)). +* Pass need_filter, has_null_map to joinRightColumns [#37256](https://github.com/ClickHouse/ClickHouse/pull/37256) ([Vladimir C](https://github.com/vdimir)). +* Activate more clangtidies [#37259](https://github.com/ClickHouse/ClickHouse/pull/37259) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix docker cleaner in workflows [#37271](https://github.com/ClickHouse/ClickHouse/pull/37271) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* tidy build fix [#37291](https://github.com/ClickHouse/ClickHouse/pull/37291) ([Alexander Gololobov](https://github.com/davenger)). +* Update run-check.py to match PR template, add comments [#37301](https://github.com/ClickHouse/ClickHouse/pull/37301) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Release without prestable [#37306](https://github.com/ClickHouse/ClickHouse/pull/37306) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fixed typos [#37322](https://github.com/ClickHouse/ClickHouse/pull/37322) ([Alexander Gololobov](https://github.com/davenger)). + diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 583ffddf279..eeb4078eb04 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.6.1.1985-stable FIXME as compared to v22.5.1.2079-stable #### Backward Incompatible Change @@ -190,3 +197,164 @@ * NO CL ENTRY: 'Revert "More parallel execution for queries with `FINAL`"'. [#38094](https://github.com/ClickHouse/ClickHouse/pull/38094) ([Alexander Tokmakov](https://github.com/tavplubix)). * NO CL ENTRY: 'Revert "Revert "add d3js based trace visualizer as gantt chart""'. [#38129](https://github.com/ClickHouse/ClickHouse/pull/38129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix collision of S3 operation log revision [#34915](https://github.com/ClickHouse/ClickHouse/pull/34915) ([ianton-ru](https://github.com/ianton-ru)). +* Change timezone in stateless tests [#35231](https://github.com/ClickHouse/ClickHouse/pull/35231) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code in ReplicatedMergeTreeRestartingThread [#36113](https://github.com/ClickHouse/ClickHouse/pull/36113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge release and performance builds [#36538](https://github.com/ClickHouse/ClickHouse/pull/36538) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add support for preprocessing ZooKeeper operations in `clickhouse-keeper` [#37036](https://github.com/ClickHouse/ClickHouse/pull/37036) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix jemalloc compatibility with LLVM libunwind [#37078](https://github.com/ClickHouse/ClickHouse/pull/37078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Separate object storage operations from disks [#37139](https://github.com/ClickHouse/ClickHouse/pull/37139) ([alesapin](https://github.com/alesapin)). +* Add test for WATCH LIMIT query in WindowView [#37219](https://github.com/ClickHouse/ClickHouse/pull/37219) ([vxider](https://github.com/Vxider)). +* Improve changelog script [#37249](https://github.com/ClickHouse/ClickHouse/pull/37249) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Rework AccessControl's notifications. [#37269](https://github.com/ClickHouse/ClickHouse/pull/37269) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add ability to pass QueryKind via clickhouse-client/local (useful for debugging) [#37290](https://github.com/ClickHouse/ClickHouse/pull/37290) ([Azat Khuzhin](https://github.com/azat)). +* Various cmake cleanups [#37300](https://github.com/ClickHouse/ClickHouse/pull/37300) ([Robert Schulze](https://github.com/rschu1ze)). +* Try to fix some trash [#37303](https://github.com/ClickHouse/ClickHouse/pull/37303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [bug-fix] root_dir is not set in copyThroughBuffers [#37319](https://github.com/ClickHouse/ClickHouse/pull/37319) ([lingo-xp](https://github.com/lingo-xp)). +* Speed up test 00157_cache_dictionary [#37320](https://github.com/ClickHouse/ClickHouse/pull/37320) ([Kruglov Pavel](https://github.com/Avogar)). +* Add changelog for 22.5 [#37339](https://github.com/ClickHouse/ClickHouse/pull/37339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release [#37346](https://github.com/ClickHouse/ClickHouse/pull/37346) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable clang-tidy readability-identifier-length [#37347](https://github.com/ClickHouse/ClickHouse/pull/37347) ([Robert Schulze](https://github.com/rschu1ze)). +* Tags and release [#37348](https://github.com/ClickHouse/ClickHouse/pull/37348) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* New stable tag [#37349](https://github.com/ClickHouse/ClickHouse/pull/37349) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fixes for eigen library build [#37369](https://github.com/ClickHouse/ClickHouse/pull/37369) ([Alexander Gololobov](https://github.com/davenger)). +* Do not fail CI if events clickhouse is down [#37371](https://github.com/ClickHouse/ClickHouse/pull/37371) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Ugly hacks for performance.tgz artifacts [#37373](https://github.com/ClickHouse/ClickHouse/pull/37373) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix docker server images building [#37374](https://github.com/ClickHouse/ClickHouse/pull/37374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix debug symbols in packages [#37379](https://github.com/ClickHouse/ClickHouse/pull/37379) ([Azat Khuzhin](https://github.com/azat)). +* Enable -Wc++98-compat-extra-semi [#37382](https://github.com/ClickHouse/ClickHouse/pull/37382) ([Robert Schulze](https://github.com/rschu1ze)). +* Mark all `operator bool()` as explicit [#37388](https://github.com/ClickHouse/ClickHouse/pull/37388) ([Anton Popov](https://github.com/CurtizJ)). +* Fixes for transactions [#37398](https://github.com/ClickHouse/ClickHouse/pull/37398) ([Alexander Tokmakov](https://github.com/tavplubix)). +* WindowTransform::moveRowNumber fix [#37400](https://github.com/ClickHouse/ClickHouse/pull/37400) ([Nikolay](https://github.com/ndchikin)). +* Fix possible memory leaks in system.certificates implementation [#37407](https://github.com/ClickHouse/ClickHouse/pull/37407) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Remove unused code in WindowView [#37426](https://github.com/ClickHouse/ClickHouse/pull/37426) ([vxider](https://github.com/Vxider)). +* Add join_algorithm='parallel_hash' to stress test [#37436](https://github.com/ClickHouse/ClickHouse/pull/37436) ([Vladimir C](https://github.com/vdimir)). +* Perf test for Norm and Distance functions for arrays and tuples [#37437](https://github.com/ClickHouse/ClickHouse/pull/37437) ([Alexander Gololobov](https://github.com/davenger)). +* FunctionBinaryRepresentation style fixes [#37438](https://github.com/ClickHouse/ClickHouse/pull/37438) ([Maksim Kita](https://github.com/kitaisreal)). +* CompressedWriteBuffer added comment [#37442](https://github.com/ClickHouse/ClickHouse/pull/37442) ([Maksim Kita](https://github.com/kitaisreal)). +* BinaryFunctionVectorized remove macro [#37447](https://github.com/ClickHouse/ClickHouse/pull/37447) ([Maksim Kita](https://github.com/kitaisreal)). +* RangeHashedDictionary added test [#37449](https://github.com/ClickHouse/ClickHouse/pull/37449) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix failed thread stateless tests [#37455](https://github.com/ClickHouse/ClickHouse/pull/37455) ([Kruglov Pavel](https://github.com/Avogar)). +* tests: fix table in 01710_projection_aggregation_in_order [#37468](https://github.com/ClickHouse/ClickHouse/pull/37468) ([Azat Khuzhin](https://github.com/azat)). +* Sync workflows paths for PR and DocsCheck [#37477](https://github.com/ClickHouse/ClickHouse/pull/37477) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Set compiler flags for ASM same as for C/CXX [#37479](https://github.com/ClickHouse/ClickHouse/pull/37479) ([Azat Khuzhin](https://github.com/azat)). +* Dynamic dispatch infrastructure style fixes [#37480](https://github.com/ClickHouse/ClickHouse/pull/37480) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactorings of LIKE/MATCH code [#37491](https://github.com/ClickHouse/ClickHouse/pull/37491) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve changelog.py script, get changelogs for 2021 [#37496](https://github.com/ClickHouse/ClickHouse/pull/37496) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to run aarch64 performance tests [#37497](https://github.com/ClickHouse/ClickHouse/pull/37497) ([Ilya Yatsishin](https://github.com/qoega)). +* Renamed arrayXXNorm/arrayXXDistance functions to XXNorm/XXDistance and fixed some overflow cases [#37502](https://github.com/ClickHouse/ClickHouse/pull/37502) ([Alexander Gololobov](https://github.com/davenger)). +* Bump cctz to 2022-05-15 [#37518](https://github.com/ClickHouse/ClickHouse/pull/37518) ([Robert Schulze](https://github.com/rschu1ze)). +* Add failed builds to the build report [#37527](https://github.com/ClickHouse/ClickHouse/pull/37527) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Functions normalizeUTF8 unstable performance tests fix [#37528](https://github.com/ClickHouse/ClickHouse/pull/37528) ([Maksim Kita](https://github.com/kitaisreal)). +* Use a separate mutex for query_factories_info in Context. [#37532](https://github.com/ClickHouse/ClickHouse/pull/37532) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update SECURITY.md [#37541](https://github.com/ClickHouse/ClickHouse/pull/37541) ([Ivan Blinkov](https://github.com/blinkov)). +* Refactor read metrics and callbacks [#37543](https://github.com/ClickHouse/ClickHouse/pull/37543) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow-up to [#37398](https://github.com/ClickHouse/ClickHouse/issues/37398) [#37547](https://github.com/ClickHouse/ClickHouse/pull/37547) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix root CMakeLists.txt search [#37552](https://github.com/ClickHouse/ClickHouse/pull/37552) ([Sergei Trifonov](https://github.com/serxa)). +* Cleanup StorageHDFS (unused variables prevent build with clang 12) [#37554](https://github.com/ClickHouse/ClickHouse/pull/37554) ([Michail Safronov](https://github.com/msaf1980)). +* Fix failed assertion in cache [#37566](https://github.com/ClickHouse/ClickHouse/pull/37566) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Slightly better jepsen tests [#37568](https://github.com/ClickHouse/ClickHouse/pull/37568) ([alesapin](https://github.com/alesapin)). +* Remove unused MergeTreeDataMergerMutator::chooseMergeAlgorithm() [#37574](https://github.com/ClickHouse/ClickHouse/pull/37574) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless files [#37579](https://github.com/ClickHouse/ClickHouse/pull/37579) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Flush and shutdown temporary table before drop [#37585](https://github.com/ClickHouse/ClickHouse/pull/37585) ([vxider](https://github.com/Vxider)). +* tests: fix 01317_no_password_in_command_line flakiness (and make it race free) [#37597](https://github.com/ClickHouse/ClickHouse/pull/37597) ([Azat Khuzhin](https://github.com/azat)). +* Use Jepsen worklow directly in PR workflow [#37599](https://github.com/ClickHouse/ClickHouse/pull/37599) ([Antonio Andelic](https://github.com/antonio2368)). +* Added LpNorm and LpDistance functions for arrays [#37601](https://github.com/ClickHouse/ClickHouse/pull/37601) ([Alexander Gololobov](https://github.com/davenger)). +* Turn on s3 tests to red mode [#37604](https://github.com/ClickHouse/ClickHouse/pull/37604) ([alesapin](https://github.com/alesapin)). +* Update FileCache.cpp [#37606](https://github.com/ClickHouse/ClickHouse/pull/37606) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix Clang-Tidy: remove std::move() from trivially-copyable object [#37609](https://github.com/ClickHouse/ClickHouse/pull/37609) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Fix clang-tidy-14, part 1 [#37612](https://github.com/ClickHouse/ClickHouse/pull/37612) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove margin in test reports and change the font [#37614](https://github.com/ClickHouse/ClickHouse/pull/37614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bugs in WindowView when using table identifier [#37619](https://github.com/ClickHouse/ClickHouse/pull/37619) ([vxider](https://github.com/Vxider)). +* Try fix tests [#37622](https://github.com/ClickHouse/ClickHouse/pull/37622) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix refactoring issue [#37629](https://github.com/ClickHouse/ClickHouse/pull/37629) ([alesapin](https://github.com/alesapin)). +* fix typo [#37637](https://github.com/ClickHouse/ClickHouse/pull/37637) ([flynn](https://github.com/ucasfl)). +* Fix excessive LIST requests to coordinator for transactions [#37640](https://github.com/ClickHouse/ClickHouse/pull/37640) ([Azat Khuzhin](https://github.com/azat)). +* Less flaky jbod rebalancer test [#37642](https://github.com/ClickHouse/ClickHouse/pull/37642) ([Amos Bird](https://github.com/amosbird)). +* Avoid useless context copy when building query interpreters [#37643](https://github.com/ClickHouse/ClickHouse/pull/37643) ([Amos Bird](https://github.com/amosbird)). +* Disable amqp-cpp and cassandra build if libuv is disabled [#37644](https://github.com/ClickHouse/ClickHouse/pull/37644) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix errors of CheckTriviallyCopyableMove type [#37647](https://github.com/ClickHouse/ClickHouse/pull/37647) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Fix hung check [#37655](https://github.com/ClickHouse/ClickHouse/pull/37655) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flaky test test_row_policy [#37658](https://github.com/ClickHouse/ClickHouse/pull/37658) ([Vitaly Baranov](https://github.com/vitlibar)). +* Trying to remove prewhere_info.alias_actions as they seem to always be included in prewhere_actions [#37661](https://github.com/ClickHouse/ClickHouse/pull/37661) ([Alexander Gololobov](https://github.com/davenger)). +* test for [#36995](https://github.com/ClickHouse/ClickHouse/issues/36995) [#37668](https://github.com/ClickHouse/ClickHouse/pull/37668) ([Denny Crane](https://github.com/den-crane)). +* Cleanup unused file [#37674](https://github.com/ClickHouse/ClickHouse/pull/37674) ([hongbin](https://github.com/xlwh)). +* more verbose sanity checks [#37676](https://github.com/ClickHouse/ClickHouse/pull/37676) ([Sergei Trifonov](https://github.com/serxa)). +* Fix `test_keeper_force_recovery*` tests [#37679](https://github.com/ClickHouse/ClickHouse/pull/37679) ([Antonio Andelic](https://github.com/antonio2368)). +* Display entires for failed tests at the top of report by default [#37704](https://github.com/ClickHouse/ClickHouse/pull/37704) ([Vladimir C](https://github.com/vdimir)). +* Make GROUPING function skip constant folding [#37710](https://github.com/ClickHouse/ClickHouse/pull/37710) ([Dmitry Novik](https://github.com/novikd)). +* Get rid of duplicate download_previous_release [#37712](https://github.com/ClickHouse/ClickHouse/pull/37712) ([Vladimir C](https://github.com/vdimir)). +* Fix possible flaky 00814_replicated_minimalistic_part_header_zookeeper [#37714](https://github.com/ClickHouse/ClickHouse/pull/37714) ([Vladimir C](https://github.com/vdimir)). +* Fix build with -DENABLE_LIBRARIES=0 [#37719](https://github.com/ClickHouse/ClickHouse/pull/37719) ([Robert Schulze](https://github.com/rschu1ze)). +* Update grpc submodule to PR 9 [#37721](https://github.com/ClickHouse/ClickHouse/pull/37721) ([Harry Lee](https://github.com/HarryLeeIBM)). +* Initialize `ParallelReadBuffer` after construction [#37726](https://github.com/ClickHouse/ClickHouse/pull/37726) ([Antonio Andelic](https://github.com/antonio2368)). +* Some fixes for tests [#37740](https://github.com/ClickHouse/ClickHouse/pull/37740) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add test 02315_pmj_union_ubsan_35857 [#37744](https://github.com/ClickHouse/ClickHouse/pull/37744) ([Vladimir C](https://github.com/vdimir)). +* Upgrade libxml2 to 2.9.14 [#37745](https://github.com/ClickHouse/ClickHouse/pull/37745) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Implement new submodule sync to fix nightly scan [#37750](https://github.com/ClickHouse/ClickHouse/pull/37750) ([Boris Kuschel](https://github.com/bkuschel)). +* Return back [#37266](https://github.com/ClickHouse/ClickHouse/issues/37266) [#37755](https://github.com/ClickHouse/ClickHouse/pull/37755) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky 01154_move_partition_long [#37763](https://github.com/ClickHouse/ClickHouse/pull/37763) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Moved `ClientConfigurationPerRequest` to ClickHouse [#37767](https://github.com/ClickHouse/ClickHouse/pull/37767) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Got rid of `S3AuthSigner` [#37769](https://github.com/ClickHouse/ClickHouse/pull/37769) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix empty target table id in WindowView [#37770](https://github.com/ClickHouse/ClickHouse/pull/37770) ([vxider](https://github.com/Vxider)). +* Typos [#37773](https://github.com/ClickHouse/ClickHouse/pull/37773) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Use multitarget framework for numZerosInTail implementation [#37777](https://github.com/ClickHouse/ClickHouse/pull/37777) ([Alexander Gololobov](https://github.com/davenger)). +* Dictionaries improve exception messages [#37778](https://github.com/ClickHouse/ClickHouse/pull/37778) ([Maksim Kita](https://github.com/kitaisreal)). +* Ignore PART_IS_TEMPORARILY_LOCKED error in BC check [#37788](https://github.com/ClickHouse/ClickHouse/pull/37788) ([Kruglov Pavel](https://github.com/Avogar)). +* Use clang compiler by default and cleanup cmake preload [#37796](https://github.com/ClickHouse/ClickHouse/pull/37796) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading of empty S3 files [#37801](https://github.com/ClickHouse/ClickHouse/pull/37801) ([Azat Khuzhin](https://github.com/azat)). +* Metadata storage abstraction for Disks [#37804](https://github.com/ClickHouse/ClickHouse/pull/37804) ([alesapin](https://github.com/alesapin)). +* Minor follow-up to cache table: std::{vector-->array} [#37806](https://github.com/ClickHouse/ClickHouse/pull/37806) ([Robert Schulze](https://github.com/rschu1ze)). +* Delay schedule of the cleaning task in WindowView [#37807](https://github.com/ClickHouse/ClickHouse/pull/37807) ([vxider](https://github.com/Vxider)). +* Fix submodule changed label condition [#37825](https://github.com/ClickHouse/ClickHouse/pull/37825) ([Vladimir C](https://github.com/vdimir)). +* Fix build (packager) [#37829](https://github.com/ClickHouse/ClickHouse/pull/37829) ([Nikita Taranov](https://github.com/nickitat)). +* Executable task forward job callback [#37830](https://github.com/ClickHouse/ClickHouse/pull/37830) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix docs check (remove git clean for yarn.lock) [#37836](https://github.com/ClickHouse/ClickHouse/pull/37836) ([Vladimir C](https://github.com/vdimir)). +* Opentracing minimal changes for processors [#37837](https://github.com/ClickHouse/ClickHouse/pull/37837) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not count covered unexpected parts on sanity check [#37839](https://github.com/ClickHouse/ClickHouse/pull/37839) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Print query in one line on fatal errors [#37840](https://github.com/ClickHouse/ClickHouse/pull/37840) ([Azat Khuzhin](https://github.com/azat)). +* Hotfix for minio in functional tests [#37845](https://github.com/ClickHouse/ClickHouse/pull/37845) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Hotfix for minio in functional tests 2 [#37846](https://github.com/ClickHouse/ClickHouse/pull/37846) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fire and clean windows in WindowView only when data is inserted [#37853](https://github.com/ClickHouse/ClickHouse/pull/37853) ([vxider](https://github.com/Vxider)). +* Don't try to kill empty list of containers in `integration/runner` [#37854](https://github.com/ClickHouse/ClickHouse/pull/37854) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Minor cleanup [#37866](https://github.com/ClickHouse/ClickHouse/pull/37866) ([Igor Nikonov](https://github.com/devcrafter)). +* Update protobuf files for kafka and rabbitmq [fix integration tests] [#37884](https://github.com/ClickHouse/ClickHouse/pull/37884) ([Nikita Taranov](https://github.com/nickitat)). +* Fix keeper converter test [#37890](https://github.com/ClickHouse/ClickHouse/pull/37890) ([alesapin](https://github.com/alesapin)). +* Fix stress hung because of attached MV with sleep() [#37892](https://github.com/ClickHouse/ClickHouse/pull/37892) ([Azat Khuzhin](https://github.com/azat)). +* Fix build [#37903](https://github.com/ClickHouse/ClickHouse/pull/37903) ([Nikita Taranov](https://github.com/nickitat)). +* Try fix `test_grpc_protocol/test.py::test_progress` [#37908](https://github.com/ClickHouse/ClickHouse/pull/37908) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable flaky tests with MaterializedPostgreSQL [#37924](https://github.com/ClickHouse/ClickHouse/pull/37924) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ProcfsMetricsProvider fix typo [#37927](https://github.com/ClickHouse/ClickHouse/pull/37927) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet check arguments size [#37930](https://github.com/ClickHouse/ClickHouse/pull/37930) ([Maksim Kita](https://github.com/kitaisreal)). +* Hierarchical dictionaries performance test fix [#37953](https://github.com/ClickHouse/ClickHouse/pull/37953) ([Maksim Kita](https://github.com/kitaisreal)). +* Normalize UTF8 performance test fix [#37954](https://github.com/ClickHouse/ClickHouse/pull/37954) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky `test_parts_delete_zookeeper` [#37957](https://github.com/ClickHouse/ClickHouse/pull/37957) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better parsing of `versionId` in `S3::URI::URI` [#37964](https://github.com/ClickHouse/ClickHouse/pull/37964) ([Vladimir Chebotarev](https://github.com/excitoon)). +* More consistent use of platform macros [#37969](https://github.com/ClickHouse/ClickHouse/pull/37969) ([Robert Schulze](https://github.com/rschu1ze)). +* Always disable --color-diagnostics for LLVM [#37970](https://github.com/ClickHouse/ClickHouse/pull/37970) ([Robert Schulze](https://github.com/rschu1ze)). +* Try fix `test_consistent_parts_after_clone_replica` [#37976](https://github.com/ClickHouse/ClickHouse/pull/37976) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Generalize setup_minio.sh [#37979](https://github.com/ClickHouse/ClickHouse/pull/37979) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add checks for numChildren in Keeper [#37980](https://github.com/ClickHouse/ClickHouse/pull/37980) ([alesapin](https://github.com/alesapin)). +* PartialSortingTransform refactoring [#37992](https://github.com/ClickHouse/ClickHouse/pull/37992) ([Maksim Kita](https://github.com/kitaisreal)). +* Follow up on self-extracting-executable [#38011](https://github.com/ClickHouse/ClickHouse/pull/38011) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update docker run client command in docs [#38015](https://github.com/ClickHouse/ClickHouse/pull/38015) ([Vladimir C](https://github.com/vdimir)). +* Add another useful `docker exec` option [#38016](https://github.com/ClickHouse/ClickHouse/pull/38016) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add release commit hash to image labels [#38020](https://github.com/ClickHouse/ClickHouse/pull/38020) ([Pavel Novitskiy](https://github.com/pnovitskiy)). +* Use correct version in `DiskObjectStorageMetadata` [#38021](https://github.com/ClickHouse/ClickHouse/pull/38021) ([Antonio Andelic](https://github.com/antonio2368)). +* Examples coroutines build fix [#38023](https://github.com/ClickHouse/ClickHouse/pull/38023) ([Maksim Kita](https://github.com/kitaisreal)). +* Use pdqsort instead of standard sort [#38025](https://github.com/ClickHouse/ClickHouse/pull/38025) ([Maksim Kita](https://github.com/kitaisreal)). +* Decimal: noexcept move constructor/assignment operator [#38026](https://github.com/ClickHouse/ClickHouse/pull/38026) ([Igor Nikonov](https://github.com/devcrafter)). +* tests: avoid "_csv.Error: field larger than field limit (131072)" error [#38030](https://github.com/ClickHouse/ClickHouse/pull/38030) ([Azat Khuzhin](https://github.com/azat)). +* More consts for disks [#38033](https://github.com/ClickHouse/ClickHouse/pull/38033) ([alesapin](https://github.com/alesapin)). +* Remove link to past event [#38045](https://github.com/ClickHouse/ClickHouse/pull/38045) ([Ivan Blinkov](https://github.com/blinkov)). +* Use workflow URL as "link" ref [#38057](https://github.com/ClickHouse/ClickHouse/pull/38057) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix a compile errors in self-extracting-executable (de)compressor [#38087](https://github.com/ClickHouse/ClickHouse/pull/38087) ([Robert Schulze](https://github.com/rschu1ze)). +* Small follow-up for FPC codec [#38089](https://github.com/ClickHouse/ClickHouse/pull/38089) ([Robert Schulze](https://github.com/rschu1ze)). +* Add dispatch to CherryPick action [#38095](https://github.com/ClickHouse/ClickHouse/pull/38095) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix: build error [#38098](https://github.com/ClickHouse/ClickHouse/pull/38098) ([Igor Nikonov](https://github.com/devcrafter)). +* Try random container name in integration tests runner [#38107](https://github.com/ClickHouse/ClickHouse/pull/38107) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try to fix fpc codec [#38117](https://github.com/ClickHouse/ClickHouse/pull/38117) ([Anton Popov](https://github.com/CurtizJ)). +* Remove unused class member [#38120](https://github.com/ClickHouse/ClickHouse/pull/38120) ([Daohui Wang](https://github.com/wangdh15)). + diff --git a/docs/changelogs/v22.6.2.12-stable.md b/docs/changelogs/v22.6.2.12-stable.md new file mode 100644 index 00000000000..224367b994a --- /dev/null +++ b/docs/changelogs/v22.6.2.12-stable.md @@ -0,0 +1,22 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.6.2.12-stable FIXME as compared to v22.6.1.1985-stable + +#### Improvement +* Backported in [#38484](https://github.com/ClickHouse/ClickHouse/issues/38484): Improve the stability for hive storage integration test. Move the data prepare step into test.py. [#38260](https://github.com/ClickHouse/ClickHouse/pull/38260) ([lgbo](https://github.com/lgbo-ustc)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#38404](https://github.com/ClickHouse/ClickHouse/issues/38404): Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove processor description from span attributes - it is not working [#38157](https://github.com/ClickHouse/ClickHouse/pull/38157) ([Ilya Yatsishin](https://github.com/qoega)). +* Checkout full repositories for performance tests [#38327](https://github.com/ClickHouse/ClickHouse/pull/38327) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to fix 02305_schema_inference_with_globs [#38337](https://github.com/ClickHouse/ClickHouse/pull/38337) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index 6da92d96335..b2f1f2448f8 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -223,7 +223,7 @@ Replication in ClickHouse can be configured on a per-table basis. You could have Replication is implemented in the `ReplicatedMergeTree` storage engine. The path in `ZooKeeper` is specified as a parameter for the storage engine. All tables with the same path in `ZooKeeper` become replicas of each other: they synchronize their data and maintain consistency. Replicas can be added and removed dynamically simply by creating or dropping a table. -Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with `ZooKeeper`, and data is replicated to all other replicas asynchronously. Because ClickHouse does not support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts, just-inserted data might be lost if one node fails. +Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with `ZooKeeper`, and data is replicated to all other replicas asynchronously. Because ClickHouse does not support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts by default, just-inserted data might be lost if one node fails. The insert quorum can be enabled using `insert_quorum` setting. Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop a partition, and so on. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the “get the part” action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. One of the leaders initiates a new merge first and writes “merge parts” actions to the log. Multiple replicas (or all) can be leaders at the same time. A replica can be prevented from becoming a leader using the `merge_tree` setting `replicated_can_become_leader`. The leaders are responsible for scheduling background merges. diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md index 5625cf3657d..83279f5f69a 100644 --- a/docs/en/development/cmake-in-clickhouse.md +++ b/docs/en/development/cmake-in-clickhouse.md @@ -349,7 +349,7 @@ Note that ClickHouse uses forks of these libraries, see https://github.com/Click Only for Linux, x86_64 or aarch64. -INSTALL_STRIPPED_BINARIES +SPLIT_DEBUG_SYMBOLS OFF Build stripped binaries with debug info in separate directory diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 3936b613bcb..8c1f6b5fc9e 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -18,6 +18,7 @@ The list of third-party libraries: | aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) | | aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) | | aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) | +| base58 | [MIT](https://github.com/ClickHouse/base-x/blob/3e58874643c087f57e82b0ff03825c933fab945a/LICENSE) | | base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) | | boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) | | boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) | @@ -40,7 +41,7 @@ The list of third-party libraries: | googletest | [BSD 3-clause](https://github.com/google/googletest/blob/e7e591764baba0a0c3c9ad0014430e7a27331d16/LICENSE) | | grpc | [Apache](https://github.com/ClickHouse-Extras/grpc/blob/60c986e15cae70aade721d26badabab1f822fdd6/LICENSE) | | h3 | [Apache](https://github.com/ClickHouse-Extras/h3/blob/c7f46cfd71fb60e2fefc90e28abe81657deff735/LICENSE) | -| hyperscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/e9f08df0213fc637aac0a5bbde9beeaeba2fe9fa/LICENSE) | +| vectorscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/73695e419c27af7fe2a099c7aa57931cc02aea5d/LICENSE) | | icu | [Public Domain](https://github.com/unicode-org/icu/blob/a56dde820dc35665a66f2e9ee8ba58e75049b668/icu4c/LICENSE) | | icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/72d9a4a7febc904e2b0a534ccb25ae40fac5f1e5/LICENSE) | | jemalloc | [BSD 2-clause](https://github.com/ClickHouse-Extras/jemalloc/blob/e6891d9746143bf2cf617493d880ba5a0b9a3efd/COPYING) | diff --git a/docs/en/engines/database-engines/index.md b/docs/en/engines/database-engines/index.md index 8e36aca695c..237112a5bee 100644 --- a/docs/en/engines/database-engines/index.md +++ b/docs/en/engines/database-engines/index.md @@ -20,6 +20,8 @@ Here is a complete list of available database engines. Follow the links for more - [PostgreSQL](../../engines/database-engines/postgresql.md) +- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) + - [Replicated](../../engines/database-engines/replicated.md) - [SQLite](../../engines/database-engines/sqlite.md) diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index cbc40993da8..4b16d877210 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -26,14 +26,30 @@ ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'passwo - `user` — MySQL user. - `password` — User password. -**Engine Settings** +## Engine Settings -- `max_rows_in_buffer` — Maximum number of rows that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `65 505`. -- `max_bytes_in_buffer` — Maximum number of bytes that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `1 048 576`. -- `max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`. -- `max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`. -- `allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`). -- `materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated. +### max_rows_in_buffer + +`max_rows_in_buffer` — Maximum number of rows that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `65 505`. + +### max_bytes_in_buffer + +`max_bytes_in_buffer` — Maximum number of bytes that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `1 048 576`. + +### max_flush_data_time + +`max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`. + +### max_wait_time_when_mysql_unavailable + +`max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`. + +### allows_query_when_mysql_lost +`allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`). + +### materialized_mysql_tables_list + +`materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated. ```sql CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***') @@ -42,12 +58,17 @@ CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', max_wait_time_when_mysql_unavailable=10000; ``` -**Settings on MySQL-server Side** +## Settings on MySQL-server Side For the correct work of `MaterializedMySQL`, there are few mandatory `MySQL`-side configuration settings that must be set: -- `default_authentication_plugin = mysql_native_password` since `MaterializedMySQL` can only authorize with this method. -- `gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializedMySQL` replication. +### default_authentication_plugin + +`default_authentication_plugin = mysql_native_password` since `MaterializedMySQL` can only authorize with this method. + +### gtid_mode + +`gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializedMySQL` replication. :::note While turning on `gtid_mode` you should also specify `enforce_gtid_consistency = on`. @@ -57,8 +78,13 @@ While turning on `gtid_mode` you should also specify `enforce_gtid_consistency = When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) tables are used with virtual `_sign` and `_version` columns. -- `_version` — Transaction counter. Type [UInt64](../../sql-reference/data-types/int-uint.md). -- `_sign` — Deletion mark. Type [Int8](../../sql-reference/data-types/int-uint.md). Possible values: +### \_version + +`_version` — Transaction counter. Type [UInt64](../../sql-reference/data-types/int-uint.md). + +### \_sign + +`_sign` — Deletion mark. Type [Int8](../../sql-reference/data-types/int-uint.md). Possible values: - `1` — Row is not deleted, - `-1` — Row is deleted. diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index 66f918f01d6..dc05c58f092 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -150,21 +150,21 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm ## Settings {#settings} -1. `materialized_postgresql_tables_list` {#materialized-postgresql-tables-list} +### `materialized_postgresql_tables_list` {#materialized-postgresql-tables-list} Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine. Default value: empty list — means whole PostgreSQL database will be replicated. -2. `materialized_postgresql_schema` {#materialized-postgresql-schema} +### `materialized_postgresql_schema` {#materialized-postgresql-schema} Default value: empty string. (Default schema is used) -3. `materialized_postgresql_schema_list` {#materialized-postgresql-schema-list} +### `materialized_postgresql_schema_list` {#materialized-postgresql-schema-list} Default value: empty list. (Default schema is used) -4. `materialized_postgresql_allow_automatic_update` {#materialized-postgresql-allow-automatic-update} +### `materialized_postgresql_allow_automatic_update` {#materialized-postgresql-allow-automatic-update} Do not use this setting before 22.1 version. @@ -177,7 +177,7 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm Default value: `0`. -5. `materialized_postgresql_max_block_size` {#materialized-postgresql-max-block-size} +### `materialized_postgresql_max_block_size` {#materialized-postgresql-max-block-size} Sets the number of rows collected in memory before flushing data into PostgreSQL database table. @@ -187,11 +187,11 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm Default value: `65536`. -6. `materialized_postgresql_replication_slot` {#materialized-postgresql-replication-slot} +### `materialized_postgresql_replication_slot` {#materialized-postgresql-replication-slot} A user-created replication slot. Must be used together with `materialized_postgresql_snapshot`. -7. `materialized_postgresql_snapshot` {#materialized-postgresql-snapshot} +### `materialized_postgresql_snapshot` {#materialized-postgresql-snapshot} A text string identifying a snapshot, from which [initial dump of PostgreSQL tables](../../engines/database-engines/materialized-postgresql.md) will be performed. Must be used together with `materialized_postgresql_replication_slot`. diff --git a/docs/en/engines/database-engines/postgresql.md b/docs/en/engines/database-engines/postgresql.md index 07181cc23e8..969a326b701 100644 --- a/docs/en/engines/database-engines/postgresql.md +++ b/docs/en/engines/database-engines/postgresql.md @@ -136,4 +136,3 @@ DESCRIBE TABLE test_database.test_table; └────────┴───────────────────┘ ``` -[Original article](https://clickhouse.com/docs/en/database-engines/postgresql/) diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md index ae32f5dd80f..9796fd73b1b 100644 --- a/docs/en/engines/table-engines/integrations/hdfs.md +++ b/docs/en/engines/table-engines/integrations/hdfs.md @@ -186,7 +186,6 @@ Similar to GraphiteMergeTree, the HDFS engine supports extended configuration us | - | - | |hadoop\_kerberos\_keytab | "" | |hadoop\_kerberos\_principal | "" | -|hadoop\_kerberos\_kinit\_command | kinit | |libhdfs3\_conf | "" | ### Limitations {#limitations} @@ -200,8 +199,7 @@ Note that due to libhdfs3 limitations only old-fashioned approach is supported, datanode communications are not secured by SASL (`HADOOP_SECURE_DN_USER` is a reliable indicator of such security approach). Use `tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh` for reference. -If `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` or `hadoop_kerberos_kinit_command` is specified, `kinit` will be invoked. `hadoop_kerberos_keytab` and `hadoop_kerberos_principal` are mandatory in this case. `kinit` tool and krb5 configuration files are required. - +If `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` or `hadoop_security_kerberos_ticket_cache_path` are specified, Kerberos authentication will be used. `hadoop_kerberos_keytab` and `hadoop_kerberos_principal` are mandatory in this case. ## HDFS Namenode HA support {#namenode-ha} libhdfs3 support HDFS namenode HA. diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 2de931c9e51..47a0e022841 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -168,7 +168,7 @@ For a list of possible configuration options, see the [librdkafka configuration ### Kerberos support {#kafka-kerberos-support} To deal with Kerberos-aware Kafka, add `security_protocol` child element with `sasl_plaintext` value. It is enough if Kerberos ticket-granting ticket is obtained and cached by OS facilities. -ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` and `sasl.kerberos.kinit.cmd` child elements. +ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab` and `sasl_kerberos_principal` child elements. Example: diff --git a/docs/en/engines/table-engines/log-family/index.md b/docs/en/engines/table-engines/log-family/index.md index 8e772341733..4ea2294554a 100644 --- a/docs/en/engines/table-engines/log-family/index.md +++ b/docs/en/engines/table-engines/log-family/index.md @@ -43,4 +43,3 @@ The `TinyLog` engine is the simplest in the family and provides the poorest func The `Log` and `StripeLog` engines support parallel data reading. When reading data, ClickHouse uses multiple threads. Each thread processes a separate data block. The `Log` engine uses a separate file for each column of the table. `StripeLog` stores all the data in one file. As a result, the `StripeLog` engine uses fewer file descriptors, but the `Log` engine provides higher efficiency when reading data. -[Original article](https://clickhouse.com/docs/en/operations/table_engines/log_family/) diff --git a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md index 5c3143c6c18..b2eea820139 100644 --- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -11,8 +11,8 @@ You can use `AggregatingMergeTree` tables for incremental data aggregation, incl The engine processes all columns with the following types: -- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) -- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) +## [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +## [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) It is appropriate to use `AggregatingMergeTree` if it reduces the number of rows by orders. @@ -36,7 +36,7 @@ For a description of request parameters, see [request description](../../../sql- **Query clauses** -When creating a `AggregatingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table. +When creating an `AggregatingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table.
diff --git a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md index afe323441ab..1b37e20d0da 100644 --- a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -7,7 +7,7 @@ sidebar_label: CollapsingMergeTree The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) and adds the logic of rows collapsing to data parts merge algorithm. -`CollapsingMergeTree` asynchronously deletes (collapses) pairs of rows if all of the fields in a sorting key (`ORDER BY`) are equivalent excepting the particular field `Sign` which can have `1` and `-1` values. Rows without a pair are kept. For more details see the [Collapsing](#table_engine-collapsingmergetree-collapsing) section of the document. +`CollapsingMergeTree` asynchronously deletes (collapses) pairs of rows if all of the fields in a sorting key (`ORDER BY`) are equivalent except the particular field `Sign`, which can have `1` and `-1` values. Rows without a pair are kept. For more details see the [Collapsing](#table_engine-collapsingmergetree-collapsing) section of the document. The engine may significantly reduce the volume of storage and increase the efficiency of `SELECT` query as a consequence. @@ -28,13 +28,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of query parameters, see [query description](../../../sql-reference/statements/create/table.md). -**CollapsingMergeTree Parameters** +## CollapsingMergeTree Parameters -- `sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. +### sign + +`sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. Column data type — `Int8`. -**Query clauses** +## Query clauses When creating a `CollapsingMergeTree` table, the same [query clauses](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) are required, as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index c1011e69ba6..9062dd3c423 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -87,10 +87,18 @@ Rollup configuration structure: ### Required Columns {#required-columns} -- `path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. -- `time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. -- `value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. Default value: `Value`. -- `version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`. +#### path_column_name + +`path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. + +#### time_column_name +`time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. + +#### value_column_name +`value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. Default value: `Value`. + +#### version_column_name +`version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`. ### Patterns {#patterns} @@ -254,7 +262,6 @@ Valid values: ``` - :::warning Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). ::: diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 15f66d2695f..3e5a0635339 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -60,57 +60,120 @@ For a description of parameters, see the [CREATE query description](../../../sql ### Query Clauses {#mergetree-query-clauses} -- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters. +#### ENGINE -- `ORDER BY` — The sorting key. +`ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters. - A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`. +#### ORDER_BY - ClickHouse uses the sorting key as a primary key if the primary key is not defined explicitly by the `PRIMARY KEY` clause. +`ORDER BY` — The sorting key. - Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). +A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`. -- `PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. In most cases you don't need partition key, and in most other cases you don't need partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead make client identifier or name the first column in the ORDER BY expression). +ClickHouse uses the sorting key as a primary key if the primary key is not defined explicitly by the `PRIMARY KEY` clause. - For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format. +Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). -- `PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional. +#### PARTITION BY - By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause. +`PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. In most cases you don't need partition key, and in most other cases you don't need partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead make client identifier or name the first column in the ORDER BY expression). -- `SAMPLE BY` — An expression for sampling. Optional. +For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format. - If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. +#### PRIMARY KEY -- `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. +`PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional. - Expression must have one `Date` or `DateTime` column as a result. Example: - `TTL date + INTERVAL 1 DAY` +By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause. - Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can be specified, but there should be no more than one `DELETE` rule. +#### SAMPLE BY - For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) +`SAMPLE BY` — An expression for sampling. Optional. -- `SETTINGS` — Additional parameters that control the behavior of the `MergeTree` (optional): +If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. - - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage). - - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage). - - `min_index_granularity_bytes` — Min allowed size of data granules in bytes. Default value: 1024b. To provide a safeguard against accidentally creating tables with very low index_granularity_bytes. See [Data Storage](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries. - - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information, see the [setting description](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) in “Server configuration parameters”. - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. - - - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with delete TTL. Default value: `14400` seconds (4 hours). - - `merge_with_recompression_ttl_timeout` — Minimum delay in seconds before repeating a merge with recompression TTL. Default value: `14400` seconds (4 hours). - - `try_fetch_recompressed_part_timeout` — Timeout (in seconds) before starting merge with recompression. During this time ClickHouse tries to fetch recompressed part from replica which assigned this merge with recompression. Default value: `7200` seconds (2 hours). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. - - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - - `storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes). - - `min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage). - - `max_parts_in_total` — Maximum number of parts in all partitions. - - `max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. You can also specify this setting in the global settings (see [max_compress_block_size](../../../operations/settings/settings.md#max-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. - - `min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. You can also specify this setting in the global settings (see [min_compress_block_size](../../../operations/settings/settings.md#min-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. - - `max_partitions_to_read` — Limits the maximum number of partitions that can be accessed in one query. You can also specify setting [max_partitions_to_read](../../../operations/settings/merge-tree-settings.md#max-partitions-to-read) in the global setting. +#### TTL + +`TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. + +Expression must have one `Date` or `DateTime` column as a result. Example: +``` +TTL date + INTERVAL 1 DAY +``` + +Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can be specified, but there should be no more than one `DELETE` rule. + +For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) + +### SETTINGS +Additional parameters that control the behavior of the `MergeTree` (optional): + +#### index_granularity + +`index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage). + +#### index_granularity_bytes + +`index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage). + +#### min_index_granularity_bytes + +`min_index_granularity_bytes` — Min allowed size of data granules in bytes. Default value: 1024b. To provide a safeguard against accidentally creating tables with very low index_granularity_bytes. See [Data Storage](#mergetree-data-storage). + +#### enable_mixed_granularity_parts + +`enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries. + +#### use_minimalistic_part_header_in_zookeeper + +`use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information, see the [setting description](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) in “Server configuration parameters”. + +#### min_merge_bytes_to_use_direct_io + +`min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. + +#### merge_with_ttl_timeout + +`merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with delete TTL. Default value: `14400` seconds (4 hours). +#### merge_with_recompression_ttl_timeout + +`merge_with_recompression_ttl_timeout` — Minimum delay in seconds before repeating a merge with recompression TTL. Default value: `14400` seconds (4 hours). + +#### try_fetch_recompressed_part_timeout + +`try_fetch_recompressed_part_timeout` — Timeout (in seconds) before starting merge with recompression. During this time ClickHouse tries to fetch recompressed part from replica which assigned this merge with recompression. Default value: `7200` seconds (2 hours). + +#### write_final_mark + +`write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. + +#### merge_max_block_size + +`merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. + +#### storage_policy + +`storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes). + +#### min_bytes_for_wide_part + +`min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage). + +#### max_parts_in_total + +`max_parts_in_total` — Maximum number of parts in all partitions. + +#### max_compress_block_size + +`max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. You can also specify this setting in the global settings (see [max_compress_block_size](../../../operations/settings/settings.md#max-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. + +#### min_compress_block_size + +`min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. You can also specify this setting in the global settings (see [min_compress_block_size](../../../operations/settings/settings.md#min-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. + +#### max_partitions_to_read + +`max_partitions_to_read` — Limits the maximum number of partitions that can be accessed in one query. You can also specify setting [max_partitions_to_read](../../../operations/settings/merge-tree-settings.md#max-partitions-to-read) in the global setting. **Example of Sections Setting** @@ -243,15 +306,29 @@ For `SELECT` queries, ClickHouse analyzes whether an index can be used. An index Thus, it is possible to quickly run queries on one or many ranges of the primary key. In this example, queries will be fast when run for a specific tracking tag, for a specific tag and date range, for a specific tag and date, for multiple tags with a date range, and so on. Let’s look at the engine configured as follows: - - ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate) SETTINGS index_granularity=8192 +```sql +ENGINE MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity=8192 +``` In this case, in queries: ``` sql -SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 -SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) -SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) +SELECT count() FROM table +WHERE EventDate = toDate(now()) +AND CounterID = 34 + +SELECT count() FROM table +WHERE EventDate = toDate(now()) +AND (CounterID = 34 OR CounterID = 42) + +SELECT count() FROM table +WHERE ((EventDate >= toDate('2014-01-01') +AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) +AND CounterID IN (101500, 731962, 160656) +AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) ``` ClickHouse will use the primary key index to trim improper data and the monthly partitioning key to trim partitions that are in improper date ranges. @@ -310,40 +387,40 @@ SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### Available Types of Indices {#available-types-of-indices} +### Available Types of Indices {#available-types-of-indices} -- `minmax` +#### `minmax` - Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of data like the primary key. +Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of data like the primary key. -- `set(max_rows)` +#### `set(max_rows)` - Stores unique values of the specified expression (no more than `max_rows` rows, `max_rows=0` means “no limits”). Uses the values to check if the `WHERE` expression is not satisfiable on a block of data. +Stores unique values of the specified expression (no more than `max_rows` rows, `max_rows=0` means “no limits”). Uses the values to check if the `WHERE` expression is not satisfiable on a block of data. -- `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` +#### `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that contains all ngrams from a block of data. Works only with datatypes: [String](../../../sql-reference/data-types/string.md), [FixedString](../../../sql-reference/data-types/fixedstring.md) and [Map](../../../sql-reference/data-types/map.md). Can be used for optimization of `EQUALS`, `LIKE` and `IN` expressions. +Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that contains all ngrams from a block of data. Works only with datatypes: [String](../../../sql-reference/data-types/string.md), [FixedString](../../../sql-reference/data-types/fixedstring.md) and [Map](../../../sql-reference/data-types/map.md). Can be used for optimization of `EQUALS`, `LIKE` and `IN` expressions. - - `n` — ngram size, - - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). - - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - - `random_seed` — The seed for Bloom filter hash functions. +- `n` — ngram size, +- `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). +- `number_of_hash_functions` — The number of hash functions used in the Bloom filter. +- `random_seed` — The seed for Bloom filter hash functions. -- `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` +#### `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters. +The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters. -- `bloom_filter([false_positive])` — Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) for the specified columns. +#### `bloom_filter([false_positive])` — Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) for the specified columns. - The optional `false_positive` parameter is the probability of receiving a false positive response from the filter. Possible values: (0, 1). Default value: 0.025. +The optional `false_positive` parameter is the probability of receiving a false positive response from the filter. Possible values: (0, 1). Default value: 0.025. - Supported data types: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`, `UUID`, `Map`. +Supported data types: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`, `UUID`, `Map`. - For `Map` data type client can specify if index should be created for keys or values using [mapKeys](../../../sql-reference/functions/tuple-map-functions.md#mapkeys) or [mapValues](../../../sql-reference/functions/tuple-map-functions.md#mapvalues) function. +For `Map` data type client can specify if index should be created for keys or values using [mapKeys](../../../sql-reference/functions/tuple-map-functions.md#mapkeys) or [mapValues](../../../sql-reference/functions/tuple-map-functions.md#mapvalues) function. - The following functions can use the filter: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall). +The following functions can use the filter: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall). - Example of index creation for `Map` data type +Example of index creation for `Map` data type ``` INDEX map_key_index mapKeys(map_column) TYPE bloom_filter GRANULARITY 1 @@ -357,7 +434,7 @@ INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100 INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 ``` -#### Functions Support {#functions-support} +### Functions Support {#functions-support} Conditions in the `WHERE` clause contains calls of the functions that operate with columns. If the column is a part of an index, ClickHouse tries to use this index when performing the functions. ClickHouse supports different subsets of functions for using indexes. @@ -466,7 +543,7 @@ The `TTL` clause can’t be used for key columns. **Examples** -Creating a table with `TTL`: +#### Creating a table with `TTL`: ``` sql CREATE TABLE example_table @@ -481,7 +558,7 @@ PARTITION BY toYYYYMM(d) ORDER BY d; ``` -Adding TTL to a column of an existing table +#### Adding TTL to a column of an existing table ``` sql ALTER TABLE example_table @@ -489,7 +566,7 @@ ALTER TABLE example_table c String TTL d + INTERVAL 1 DAY; ``` -Altering TTL of the column +#### Altering TTL of the column ``` sql ALTER TABLE example_table @@ -524,7 +601,7 @@ If a column is not part of the `GROUP BY` expression and is not set explicitly i **Examples** -Creating a table with `TTL`: +#### Creating a table with `TTL`: ``` sql CREATE TABLE example_table @@ -540,7 +617,7 @@ TTL d + INTERVAL 1 MONTH [DELETE], d + INTERVAL 2 WEEK TO DISK 'bbb'; ``` -Altering `TTL` of the table: +#### Altering `TTL` of the table: ``` sql ALTER TABLE example_table @@ -561,7 +638,7 @@ ORDER BY d TTL d + INTERVAL 1 MONTH DELETE WHERE toDayOfWeek(d) = 1; ``` -Creating a table, where expired rows are recompressed: +#### Creating a table, where expired rows are recompressed: ```sql CREATE TABLE table_for_recompression diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md index 5586d108ead..daa507cab66 100644 --- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,16 +33,18 @@ For a description of request parameters, see [statement description](../../../sq Uniqueness of rows is determined by the `ORDER BY` table section, not `PRIMARY KEY`. ::: -**ReplacingMergeTree Parameters** +## ReplacingMergeTree Parameters -- `ver` — column with the version number. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter. +### ver + +`ver` — column with the version number. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter. When merging, `ReplacingMergeTree` from all the rows with the same sorting key leaves only one: - The last in the selection, if `ver` not set. A selection is a set of rows in a set of parts participating in the merge. The most recently created part (the last insert) will be the last one in the selection. Thus, after deduplication, the very last row from the most recent insert will remain for each unique sorting key. - With the maximum version, if `ver` specified. If `ver` is the same for several rows, then it will use "if `ver` is not specified" rule for them, i.e. the most recent inserted row will remain. -**Query clauses** +## Query clauses When creating a `ReplacingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 3562bdf6d3a..0dfcdccb029 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -120,11 +120,19 @@ The system monitors data synchronicity on replicas and is able to recover after The `Replicated` prefix is added to the table engine name. For example:`ReplicatedMergeTree`. -**Replicated\*MergeTree parameters** +### Replicated\*MergeTree parameters -- `zoo_path` — The path to the table in ClickHouse Keeper. -- `replica_name` — The replica name in ClickHouse Keeper. -- `other_parameters` — Parameters of an engine which is used for creating the replicated version, for example, version in `ReplacingMergeTree`. +#### zoo_path + +`zoo_path` — The path to the table in ClickHouse Keeper. + +#### replica_name + +`replica_name` — The replica name in ClickHouse Keeper. + +#### other_parameters + +`other_parameters` — Parameters of an engine which is used for creating the replicated version, for example, version in `ReplacingMergeTree`. Example: diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index b532aef1980..7afa7cf028e 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -26,14 +26,16 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of request parameters, see [request description](../../../sql-reference/statements/create/table.md). -**Parameters of SummingMergeTree** +### Parameters of SummingMergeTree -- `columns` - a tuple with the names of columns where values will be summarized. Optional parameter. +#### columns + +`columns` - a tuple with the names of columns where values will be summarized. Optional parameter. The columns must be of a numeric type and must not be in the primary key. If `columns` not specified, ClickHouse summarizes the values in all columns with a numeric data type that are not in the primary key. -**Query clauses** +### Query clauses When creating a `SummingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index c1fe5dfffdf..5642602f4a1 100644 --- a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -31,21 +31,25 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of query parameters, see the [query description](../../../sql-reference/statements/create/table.md). -**Engine Parameters** +### Engine Parameters ``` sql VersionedCollapsingMergeTree(sign, version) ``` -- `sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. +#### sign + +`sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. The column data type should be `Int8`. -- `version` — Name of the column with the version of the object state. +#### version + +`version` — Name of the column with the version of the object state. The column data type should be `UInt*`. -**Query Clauses** +### Query Clauses When creating a `VersionedCollapsingMergeTree` table, the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index 5f81bd76ae4..bcd7c390eb1 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -11,24 +11,45 @@ Buffers the data to write in RAM, periodically flushing it to another table. Dur Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) ``` -Engine parameters: +### Engine parameters: -- `database` – Database name. You can use `currentDatabase()` or another constant expression that returns a string. -- `table` – Table to flush data to. -- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16. -- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, and `max_bytes` – Conditions for flushing data from the buffer. +#### database -Optional engine parameters: +`database` – Database name. You can use `currentDatabase()` or another constant expression that returns a string. -- `flush_time`, `flush_rows`, `flush_bytes` – Conditions for flushing data from the buffer, that will happen only in background (omitted or zero means no `flush*` parameters). +#### table + +`table` – Table to flush data to. + +#### num_layers + +`num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16. + +#### min_time, max_time, min_rows, max_rows, min_bytes, and max_bytes + +Conditions for flushing data from the buffer. + +### Optional engine parameters: + +#### flush_time, flush_rows, and flush_bytes + +Conditions for flushing data from the buffer, that will happen only in background (omitted or zero means no `flush*` parameters). Data is flushed from the buffer and written to the destination table if all the `min*` conditions or at least one `max*` condition are met. Also, if at least one `flush*` condition are met flush initiated in background, this is different from `max*`, since `flush*` allows you to configure background flushes separately to avoid adding latency for `INSERT` (into `Buffer`) queries. -- `min_time`, `max_time`, `flush_time` – Condition for the time in seconds from the moment of the first write to the buffer. -- `min_rows`, `max_rows`, `flush_rows` – Condition for the number of rows in the buffer. -- `min_bytes`, `max_bytes`, `flush_bytes` – Condition for the number of bytes in the buffer. +#### min_time, max_time, and flush_time + +Condition for the time in seconds from the moment of the first write to the buffer. + +#### min_rows, max_rows, and flush_rows + +Condition for the number of rows in the buffer. + +#### min_bytes, max_bytes, and flush_bytes + +Condition for the number of bytes in the buffer. During the write operation, data is inserted to a `num_layers` number of random buffers. Or, if the data part to insert is large enough (greater than `max_rows` or `max_bytes`), it is written directly to the destination table, omitting the buffer. diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index fe3348c4d78..d643d4b3c68 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -27,42 +27,70 @@ When the `Distributed` table is pointing to a table on the current server you ca CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] AS [db2.]name2 ENGINE = Distributed(cluster, database, table[, sharding_key[, policy_name]]) [SETTINGS name=value, ...] ``` -**Distributed Parameters** +### Distributed Parameters -- `cluster` - the cluster name in the server’s config file +#### cluster -- `database` - the name of a remote database +`cluster` - the cluster name in the server’s config file -- `table` - the name of a remote table +#### database -- `sharding_key` - (optionally) sharding key +`database` - the name of a remote database -- `policy_name` - (optionally) policy name, it will be used to store temporary files for async send +#### table + +`table` - the name of a remote table + +#### sharding_key + +`sharding_key` - (optionally) sharding key + +#### policy_name + +`policy_name` - (optionally) policy name, it will be used to store temporary files for async send **See Also** - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) setting - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) for the examples -**Distributed Settings** +### Distributed Settings -- `fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. +#### fsync_after_insert -- `fsync_directories` - do the `fsync` for directories. Guarantees that the OS refreshed directory metadata after operations related to asynchronous inserts on Distributed table (after insert, after sending the data to shard, etc). +`fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. -- `bytes_to_throw_insert` - if more than this number of compressed bytes will be pending for async INSERT, an exception will be thrown. 0 - do not throw. Default 0. +#### fsync_directories -- `bytes_to_delay_insert` - if more than this number of compressed bytes will be pending for async INSERT, the query will be delayed. 0 - do not delay. Default 0. +`fsync_directories` - do the `fsync` for directories. Guarantees that the OS refreshed directory metadata after operations related to asynchronous inserts on Distributed table (after insert, after sending the data to shard, etc). -- `max_delay_to_insert` - max delay of inserting data into Distributed table in seconds, if there are a lot of pending bytes for async send. Default 60. +#### bytes_to_throw_insert -- `monitor_batch_inserts` - same as [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) +`bytes_to_throw_insert` - if more than this number of compressed bytes will be pending for async INSERT, an exception will be thrown. 0 - do not throw. Default 0. -- `monitor_split_batch_on_failure` - same as [distributed_directory_monitor_split_batch_on_failure](../../../operations/settings/settings.md#distributed_directory_monitor_split_batch_on_failure) +#### bytes_to_delay_insert -- `monitor_sleep_time_ms` - same as [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) +`bytes_to_delay_insert` - if more than this number of compressed bytes will be pending for async INSERT, the query will be delayed. 0 - do not delay. Default 0. -- `monitor_max_sleep_time_ms` - same as [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) +#### max_delay_to_insert + +`max_delay_to_insert` - max delay of inserting data into Distributed table in seconds, if there are a lot of pending bytes for async send. Default 60. + +#### monitor_batch_inserts + +`monitor_batch_inserts` - same as [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) + +#### monitor_split_batch_on_failure + +`monitor_split_batch_on_failure` - same as [distributed_directory_monitor_split_batch_on_failure](../../../operations/settings/settings.md#distributed_directory_monitor_split_batch_on_failure) + +#### monitor_sleep_time_ms + +`monitor_sleep_time_ms` - same as [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) + +#### monitor_max_sleep_time_ms + +`monitor_max_sleep_time_ms` - same as [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) :::note **Durability settings** (`fsync_...`): @@ -213,7 +241,9 @@ To learn more about how distibuted `in` and `global in` queries are processed, r ## Virtual Columns {#virtual-columns} -- `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). +#### _shard_num + +`_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). :::note Since [remote](../../../sql-reference/table-functions/remote.md) and [cluster](../../../sql-reference/table-functions/cluster.md) table functions internally create temporary Distributed table, `_shard_num` is available there too. diff --git a/docs/en/engines/table-engines/special/file.md b/docs/en/engines/table-engines/special/file.md index 5f27bc73e1d..7a53670bebd 100644 --- a/docs/en/engines/table-engines/special/file.md +++ b/docs/en/engines/table-engines/special/file.md @@ -86,4 +86,3 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 - Indices - Replication -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/file/) diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md index bb9744103f7..4e628b8b9b0 100644 --- a/docs/en/engines/table-engines/special/join.md +++ b/docs/en/engines/table-engines/special/join.md @@ -23,11 +23,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] See the detailed description of the [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query) query. -**Engine Parameters** +## Engine Parameters -- `join_strictness` – [JOIN strictness](../../../sql-reference/statements/select/join.md#select-join-types). -- `join_type` – [JOIN type](../../../sql-reference/statements/select/join.md#select-join-types). -- `k1[, k2, ...]` – Key columns from the `USING` clause that the `JOIN` operation is made with. +### join_strictness + +`join_strictness` – [JOIN strictness](../../../sql-reference/statements/select/join.md#select-join-types). + +### join_type + +`join_type` – [JOIN type](../../../sql-reference/statements/select/join.md#select-join-types). + +### Key columns + +`k1[, k2, ...]` – Key columns from the `USING` clause that the `JOIN` operation is made with. Enter `join_strictness` and `join_type` parameters without quotes, for example, `Join(ANY, LEFT, col1)`. They must match the `JOIN` operation that the table will be used for. If the parameters do not match, ClickHouse does not throw an exception and may return incorrect data. @@ -56,12 +64,28 @@ Main use-cases for `Join`-engine tables are following: When creating a table, the following settings are applied: -- [join_use_nulls](../../../operations/settings/settings.md#join_use_nulls) -- [max_rows_in_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) -- [max_bytes_in_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) -- [join_overflow_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) -- [join_any_take_last_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) -- [persistent](../../../operations/settings/settings.md#persistent) +#### join_use_nulls + +[join_use_nulls](../../../operations/settings/settings.md#join_use_nulls) + +#### max_rows_in_join + +[max_rows_in_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) + +#### max_bytes_in_join + +[max_bytes_in_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) + +#### join_overflow_mode + +[join_overflow_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) + +#### join_any_take_last_row + +[join_any_take_last_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) +#### join_use_nulls + +[persistent](../../../operations/settings/settings.md#persistent) The `Join`-engine tables can’t be used in `GLOBAL JOIN` operations. @@ -127,4 +151,3 @@ ALTER TABLE id_val_join DELETE WHERE id = 3; └────┴─────┘ ``` -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/join/) diff --git a/docs/en/engines/table-engines/special/merge.md b/docs/en/engines/table-engines/special/merge.md index 0f97acda8b5..d32547a300c 100644 --- a/docs/en/engines/table-engines/special/merge.md +++ b/docs/en/engines/table-engines/special/merge.md @@ -15,14 +15,18 @@ Reading is automatically parallelized. Writing to a table is not supported. When CREATE TABLE ... Engine=Merge(db_name, tables_regexp) ``` -**Engine Parameters** +## Engine Parameters -- `db_name` — Possible values: +### db_name + +`db_name` — Possible values: - database name, - constant expression that returns a string with a database name, for example, `currentDatabase()`, - `REGEXP(expression)`, where `expression` is a regular expression to match the DB names. -- `tables_regexp` — A regular expression to match the table names in the specified DB or DBs. +### tables_regexp + +`tables_regexp` — A regular expression to match the table names in the specified DB or DBs. Regular expressions — [re2](https://github.com/google/re2) (supports a subset of PCRE), case-sensitive. See the notes about escaping symbols in regular expressions in the "match" section. @@ -82,4 +86,3 @@ SELECT * FROM WatchLog; - [Virtual columns](../../../engines/table-engines/special/index.md#table_engines-virtual_columns) - [merge](../../../sql-reference/table-functions/merge.md) table function -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/merge/) diff --git a/docs/en/engines/table-engines/special/null.md b/docs/en/engines/table-engines/special/null.md index 5e775227dab..ca02d8e300b 100644 --- a/docs/en/engines/table-engines/special/null.md +++ b/docs/en/engines/table-engines/special/null.md @@ -10,6 +10,3 @@ When writing to a `Null` table, data is ignored. When reading from a `Null` tabl :::note If you are wondering why this is useful, note that you can create a materialized view on a `Null` table. So the data written to the table will end up affecting the view, but original raw data will still be discarded. ::: - - -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/null/) diff --git a/docs/en/engines/table-engines/special/set.md b/docs/en/engines/table-engines/special/set.md index 46e31af7ff1..f7114f04cea 100644 --- a/docs/en/engines/table-engines/special/set.md +++ b/docs/en/engines/table-engines/special/set.md @@ -20,4 +20,3 @@ When creating a table, the following settings are applied: - [persistent](../../../operations/settings/settings.md#persistent) -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/set/) diff --git a/docs/en/engines/table-engines/special/url.md b/docs/en/engines/table-engines/special/url.md index 19246b82219..82617e9425d 100644 --- a/docs/en/engines/table-engines/special/url.md +++ b/docs/en/engines/table-engines/special/url.md @@ -89,4 +89,3 @@ SELECT * FROM url_engine_table - Indexes. - Replication. -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/url/) diff --git a/docs/en/getting-started/example-datasets/brown-benchmark.md b/docs/en/getting-started/example-datasets/brown-benchmark.md index 8da7a18bac3..cd4f5ae4a6b 100644 --- a/docs/en/getting-started/example-datasets/brown-benchmark.md +++ b/docs/en/getting-started/example-datasets/brown-benchmark.md @@ -455,4 +455,3 @@ ORDER BY yr, The data is also available for interactive queries in the [Playground](https://play.clickhouse.com/play?user=play), [example](https://play.clickhouse.com/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==). -[Original article](https://clickhouse.com/docs/en/getting_started/example_datasets/brown-benchmark/) diff --git a/docs/en/operations/access-rights.md b/docs/en/operations/access-rights.md index 34d79aa99d9..a431f10fbad 100644 --- a/docs/en/operations/access-rights.md +++ b/docs/en/operations/access-rights.md @@ -1,10 +1,9 @@ --- sidebar_position: 48 sidebar_label: Access Control and Account Management +title: Access Control and Account Management --- -# Access Control and Account Management - ClickHouse supports access control management based on [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) approach. ClickHouse access entities: diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index af2ba713ec1..d358267c4f0 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -13,4 +13,3 @@ The following external authenticators and directories are supported: - Kerberos [Authenticator](./kerberos.md#external-authenticators-kerberos) - [SSL X.509 authentication](./ssl-x509.md#ssl-external-authentication) -[Original article](https://clickhouse.com/docs/en/operations/external-authenticators/index/) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index ad879679a3d..d3a50969a39 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -193,6 +193,35 @@ Sets the delay before remove table data in seconds. If the query has `SYNC` modi Default value: `480` (8 minute). +## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec} + +Parameter of a task that cleans up garbage from `store/` directory. +If some subdirectory is not used by clickhouse-server and this directory was not modified for last +`database_catalog_unused_dir_hide_timeout_sec` seconds, the task will "hide" this directory by +removing all access rights. It also works for directories that clickhouse-server does not +expect to see inside `store/`. Zero means "immediately". + +Default value: `3600` (1 hour). + +## database_catalog_unused_dir_rm_timeout_sec {#database_catalog_unused_dir_rm_timeout_sec} + +Parameter of a task that cleans up garbage from `store/` directory. +If some subdirectory is not used by clickhouse-server and it was previousely "hidden" +(see [database_catalog_unused_dir_hide_timeout_sec](../../operations/server-configuration-parameters/settings.md#database_catalog_unused_dir_hide_timeout_sec)) +and this directory was not modified for last +`database_catalog_unused_dir_rm_timeout_sec` seconds, the task will remove this directory. +It also works for directories that clickhouse-server does not +expect to see inside `store/`. Zero means "never". + +Default value: `2592000` (30 days). + +## database_catalog_unused_dir_cleanup_period_sec {#database_catalog_unused_dir_cleanup_period_sec} + +Parameter of a task that cleans up garbage from `store/` directory. +Sets scheduling period of the task. Zero means "never". + +Default value: `86400` (1 day). + ## default_database {#default-database} The default database. diff --git a/docs/en/operations/system-tables/distributed_ddl_queue.md b/docs/en/operations/system-tables/distributed_ddl_queue.md index 5a2478b22d9..a35d4a2a5b7 100644 --- a/docs/en/operations/system-tables/distributed_ddl_queue.md +++ b/docs/en/operations/system-tables/distributed_ddl_queue.md @@ -61,4 +61,3 @@ exception_code: ZOK 2 rows in set. Elapsed: 0.025 sec. ``` -[Original article](https://clickhouse.com/docs/en/operations/system_tables/distributed_ddl_queuedistributed_ddl_queue.md) diff --git a/docs/en/operations/system-tables/distribution_queue.md b/docs/en/operations/system-tables/distribution_queue.md index 88d376c7553..896491a458b 100644 --- a/docs/en/operations/system-tables/distribution_queue.md +++ b/docs/en/operations/system-tables/distribution_queue.md @@ -47,4 +47,3 @@ last_exception: - [Distributed table engine](../../engines/table-engines/special/distributed.md) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/distribution_queue) diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md index 1b3b97af019..a9ca32ae030 100644 --- a/docs/en/operations/system-tables/opentelemetry_span_log.md +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -50,4 +50,3 @@ attribute.values: [] - [OpenTelemetry](../../operations/opentelemetry.md) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/opentelemetry_span_log) diff --git a/docs/en/operations/system-tables/parts_columns.md b/docs/en/operations/system-tables/parts_columns.md index 0439da79ab3..2f85b912f38 100644 --- a/docs/en/operations/system-tables/parts_columns.md +++ b/docs/en/operations/system-tables/parts_columns.md @@ -145,4 +145,3 @@ column_marks_bytes: 48 - [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/parts_columns) diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md index cb22345c3a2..a7ac748ebbd 100644 --- a/docs/en/operations/system-tables/replication_queue.md +++ b/docs/en/operations/system-tables/replication_queue.md @@ -88,4 +88,3 @@ last_postpone_time: 1970-01-01 03:00:00 - [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md#query-language-system-replicated) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/replication_queue) diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md index f364bc85088..5325311a9e6 100644 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -128,7 +128,8 @@ You should never use manually written scripts to transfer data between different If you want to divide an existing ZooKeeper cluster into two, the correct way is to increase the number of its replicas and then reconfigure it as two independent clusters. -You can run ClickHouse Keeper on the same server as ClickHouse, but do not run ZooKeeper on the same servers as ClickHouse. Because ZooKeeper is very sensitive for latency and ClickHouse may utilize all available system resources. +You can run ClickHouse Keeper on the same server as ClickHouse in test environments, or in environments with low ingestion rate. +For production environments we suggest to use separate servers for ClickHouse and ZooKeeper/Keeper, or place ClickHouse files and Keeper files on to separate disks. Because ZooKeeper/Keeper are very sensitive for disk latency and ClickHouse may utilize all available system resources. You can have ZooKeeper observers in an ensemble but ClickHouse servers should not interact with observers. diff --git a/docs/en/sql-reference/aggregate-functions/reference/meanztest.md b/docs/en/sql-reference/aggregate-functions/reference/meanztest.md index d129e5722bc..0752df05818 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/meanztest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/meanztest.md @@ -66,5 +66,3 @@ Result: └──────────────────────────────────────────────────────────────────────────────────┘ ``` - -[Original article](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/meanZTest/) diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index 82c09ed606e..0a0278f970e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -69,4 +69,3 @@ Result: - [Welch's t-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) - [studentTTest function](studentttest.md#studentttest) -[Original article](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/welchTTest/) diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md index e67f86743a9..ddc71e75e44 100644 --- a/docs/en/sql-reference/data-types/datetime64.md +++ b/docs/en/sql-reference/data-types/datetime64.md @@ -7,7 +7,7 @@ sidebar_label: DateTime64 Allows to store an instant in time, that can be expressed as a calendar date and a time of a day, with defined sub-second precision -Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. Typically are used - 3 (milliseconds), 6 (microseconds), 9 (nanoseconds). **Syntax:** @@ -34,7 +34,7 @@ ENGINE = TinyLog; ``` ``` sql -INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2); +INSERT INTO dt Values (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3); ``` ``` sql @@ -43,12 +43,13 @@ SELECT * FROM dt; ``` text ┌───────────────timestamp─┬─event_id─┐ -│ 2019-01-01 03:00:00.000 │ 1 │ -│ 2019-01-01 00:00:00.000 │ 2 │ +│ 2019-01-01 03:00:00.123 │ 1 │ +│ 2019-01-01 03:00:00.123 │ 2 │ +│ 2019-01-01 00:00:00.000 │ 3 │ └─────────────────────────┴──────────┘ ``` -- When inserting datetime as an integer, it is treated as an appropriately scaled Unix Timestamp (UTC). `1546300800000` (with precision 3) represents `'2019-01-01 00:00:00'` UTC. However, as `timestamp` column has `Asia/Istanbul` (UTC+3) timezone specified, when outputting as a string the value will be shown as `'2019-01-01 03:00:00'`. +- When inserting datetime as an integer, it is treated as an appropriately scaled Unix Timestamp (UTC). `1546300800000` (with precision 3) represents `'2019-01-01 00:00:00'` UTC. However, as `timestamp` column has `Asia/Istanbul` (UTC+3) timezone specified, when outputting as a string the value will be shown as `'2019-01-01 03:00:00'`. Inserting datetime as a decimal will treat it similarly as an integer, except the value before the decimal point is the Unix Timestamp up to and including the seconds, and after the decimal point will be treated as the precision. - When inserting string value as datetime, it is treated as being in column timezone. `'2019-01-01 00:00:00'` will be treated as being in `Asia/Istanbul` timezone and stored as `1546290000000`. 2. Filtering on `DateTime64` values @@ -65,6 +66,20 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/ Unlike `DateTime`, `DateTime64` values are not converted from `String` automatically. +``` sql +SELECT * FROM dt WHERE timestamp = toDateTime64(1546300800.123, 3); +``` + +``` text +┌───────────────timestamp─┬─event_id─┐ +│ 2019-01-01 00:00:00.123 │ 1 │ +│ 2019-01-01 00:00:00.123 │ 2 │ +└─────────────────────────┴──────────┘ +``` + +Contrary to inserting, the `toDateTime64` function will treat all values as the decimal variant, so precision needs to +be given after the decimal point. + 3. Getting a time zone for a `DateTime64`-type value: ``` sql diff --git a/docs/en/sql-reference/data-types/domains/index.md b/docs/en/sql-reference/data-types/domains/index.md index f9bd6eea07e..50599db2f47 100644 --- a/docs/en/sql-reference/data-types/domains/index.md +++ b/docs/en/sql-reference/data-types/domains/index.md @@ -27,4 +27,3 @@ You can use domains anywhere corresponding base type can be used, for example: - Can’t implicitly convert string values into domain values when inserting data from another column or table. - Domain adds no constrains on stored values. -[Original article](https://clickhouse.com/docs/en/data_types/domains/) diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index c8edf985582..22fc56dbcf5 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -104,4 +104,3 @@ Result: └─────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────┘ ``` -[Original article](https://clickhouse.com/docs/en/data-types/geo/) diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index e913a5f34e3..65a0f9cbc52 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -108,4 +108,3 @@ Result: - [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function - [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function -[Original article](https://clickhouse.com/docs/en/data-types/map/) diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index d0f604e8d8e..069e2e68671 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -39,4 +39,3 @@ Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way CREATE TABLE simple (id UInt64, val SimpleAggregateFunction(sum, Double)) ENGINE=AggregatingMergeTree ORDER BY id; ``` -[Original article](https://clickhouse.com/docs/en/data_types/simpleaggregatefunction/) diff --git a/docs/en/sql-reference/functions/encryption-functions.md b/docs/en/sql-reference/functions/encryption-functions.md index 5b3fe2201e6..75f6cf18766 100644 --- a/docs/en/sql-reference/functions/encryption-functions.md +++ b/docs/en/sql-reference/functions/encryption-functions.md @@ -355,4 +355,3 @@ Result: └───────────┘ ``` -[Original article](https://clickhouse.com/docs/en/sql-reference/functions/encryption_functions/) diff --git a/docs/en/sql-reference/functions/geo/index.md b/docs/en/sql-reference/functions/geo/index.md index f76c3a3f731..c0162cb5b63 100644 --- a/docs/en/sql-reference/functions/geo/index.md +++ b/docs/en/sql-reference/functions/geo/index.md @@ -3,6 +3,74 @@ sidebar_label: Geo sidebar_position: 62 --- -# Geo Functions +# Geo Functions + +## Geographical Coordinates Functions + +- [greatCircleDistance](./coordinates.md#greatCircleDistance) +- [geoDistance](./coordinates.md#geoDistance) +- [greatCircleAngle](./coordinates.md#greatCircleAngle) +- [pointInEllipses](./coordinates.md#pointInEllipses) +- [pointInPolygon](./coordinates.md#pointInPolygon) + +## Geohash Functions +- [geohashEncode](./geohash.md#geohashEncode) +- [geohashDecode](./geohash.md#geohashDecode) +- [geohashesInBox](./geohash.md#geohashesInBox) + +## H3 Indexes Functions + +- [h3IsValid](./h3.md#h3IsValid) +- [h3GetResolution](./h3.md#h3GetResolution) +- [h3EdgeAngle](./h3.md#h3EdgeAngle) +- [h3EdgeLengthM​](./h3.md#h3EdgeLengthM​) +- [h3EdgeLengthKm](./h3.md#h3EdgeLengthKm) +- [geoToH3](./h3.md#geoToH3) +- [h3ToGeo](./h3.md#h3ToGeo) +- [h3ToGeoBoundary](./h3.md#h3ToGeoBoundary) +- [h3kRing](./h3.md#h3kRing) +- [h3GetBaseCell](./h3.md#h3GetBaseCell) +- [h3HexAreaM2](./h3.md#h3HexAreaM2) +- [h3HexAreaKm2](./h3.md#h3HexAreaKm2) +- [h3IndexesAreNeighbors](./h3.md#h3IndexesAreNeighbors) +- [h3ToChildren](./h3.md#h3ToChildren) +- [h3ToParent](./h3.md#h3ToParent) +- [h3ToString](./h3.md#h3ToString) +- [stringToH3](./h3.md#stringToH3) +- [h3GetResolution](./h3.md#h3GetResolution) +- [h3IsResClassIII](./h3.md#h3IsResClassIII) +- [h3IsPentagon](./h3.md#h3IsPentagon) +- [h3GetFaces](./h3.md#h3GetFaces) +- [h3CellAreaM2](./h3.md#h3CellAreaM2) +- [h3CellAreaRads2](./h3.md#h3CellAreaRads2) +- [h3ToCenterChild](./h3.md#h3ToCenterChild) +- [h3ExactEdgeLengthM](./h3.md#h3ExactEdgeLengthM) +- [h3ExactEdgeLengthKm](./h3.md#h3ExactEdgeLengthKm) +- [h3ExactEdgeLengthRads](./h3.md#h3ExactEdgeLengthRads) +- [h3NumHexagons](./h3.md#h3NumHexagons) +- [h3Line](./h3.md#h3Line) +- [h3Distance](./h3.md#h3Distance) +- [h3HexRing](./h3.md#h3HexRing) +- [h3GetUnidirectionalEdge](./h3.md#h3GetUnidirectionalEdge) +- [h3UnidirectionalEdgeIsValid](./h3.md#h3UnidirectionalEdgeIsValid) +- [h3GetOriginIndexFromUnidirectionalEdge](./h3.md#h3GetOriginIndexFromUnidirectionalEdge) +- [h3GetDestinationIndexFromUnidirectionalEdge](./h3.md#h3GetDestinationIndexFromUnidirectionalEdge) +- [h3GetIndexesFromUnidirectionalEdge](./h3.md#h3GetIndexesFromUnidirectionalEdge) +- [h3GetUnidirectionalEdgesFromHexagon](./h3.md#h3GetUnidirectionalEdgesFromHexagon) +- [h3GetUnidirectionalEdgeBoundary](./h3.md#h3GetUnidirectionalEdgeBoundary) + +## S2 Index Functions + +- [geoToS2](./s2.md#geoToS2) +- [s2ToGeo](./s2.md#s2ToGeo) +- [s2GetNeighbors](./s2.md#s2GetNeighbors) +- [s2CellsIntersect](./s2.md#s2CellsIntersect) +- [s2CapContains](./s2.md#s2CapContains) +- [s2CapUnion](./s2.md#s2CapUnion) +- [s2RectAdd](./s2.md#s2RectAdd) +- [s2RectContains](./s2.md#s2RectContains) +- [s2RectUinion](./s2.md#s2RectUinion) +- [s2RectIntersection](./s2.md#s2RectIntersection) + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index cac546d9a70..38d247457cf 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -494,6 +494,48 @@ If the ‘s’ string is non-empty and does not contain the ‘c’ character at Returns the string ‘s’ that was converted from the encoding in ‘from’ to the encoding in ‘to’. +## Base58Encode(plaintext[, alphabet_name]), Base58Decode(encoded_text[, alphabet_name]) + +Accepts a String and encodes/decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using specified alphabet. + +**Syntax** + +```sql +base58Encode(decoded[, alphabet_name]) +base58Decode(encoded[, alphabet_name]) +``` + +**Arguments** + +- `decoded` — [String](../../sql-reference/data-types/string.md) column or constant. +- `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, an exception is thrown. +- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `bitcoin`. + +**Returned value** + +- A string containing encoded/decoded value of 1st argument. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT base58Encode('encode', 'flickr'); +SELECT base58Decode('izCFiDUY', 'ripple'); +``` + +Result: +```text +┌─base58Encode('encode', 'flickr')─┐ +│ SvyTHb1D │ +└──────────────────────────────────┘ +┌─base58Decode('izCFiDUY', 'ripple')─┐ +│ decode │ +└────────────────────────────────────┘ +``` + ## base64Encode(s) Encodes ‘s’ string into base64 diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index e2afddeaaf3..3f4db831e3d 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -301,6 +301,80 @@ Result: └─────────────────────────────────────────────────────────┴───────────────────────────────────────────────────────────┘ ``` +## toDateTime64 + +Converts the argument to the [DateTime64](../../sql-reference/data-types/datetime64.md) data type. + +**Syntax** + +``` sql +toDateTime64(expr, scale, [timezone]) +``` + +**Arguments** + +- `expr` — The value. [String](../../sql-reference/data-types/string.md), [UInt32](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [DateTime](../../sql-reference/data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` - Time zone of the specified datetime64 object. + +**Returned value** + +- A calendar date and time of day, with sub-second precision. + +Type: [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +1. The value is within the range: + +``` sql +SELECT toDateTime64('1955-01-01 00:00:00.000', 3) AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64('1955-01-01 00:00:00.000', 3))─┐ +│ 1955-01-01 00:00:00.000 │ DateTime64(3) │ +└─────────────────────────┴────────────────────────────────────────────────────────┘ +``` + +2. As decimal with precision: + +``` sql +SELECT toDateTime64(1546300800.000, 3) AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64(1546300800., 3))─┐ +│ 2019-01-01 00:00:00.000 │ DateTime64(3) │ +└─────────────────────────┴──────────────────────────────────────────┘ +``` + +Without the decimal point the value is still treated as Unix Timestamp in seconds: + +``` sql +SELECT toDateTime64(1546300800000, 3) AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64(1546300800000, 3))─┐ +│ 2282-12-31 00:00:00.000 │ DateTime64(3) │ +└─────────────────────────┴────────────────────────────────────────────┘ +``` + + +3. With `timezone`: + +``` sql +SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul'))─┐ +│ 2019-01-01 00:00:00.000 │ DateTime64(3, 'Asia/Istanbul') │ +└─────────────────────────┴─────────────────────────────────────────────────────────────────────┘ +``` + + ## toDecimal(32\|64\|128\|256) Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) data type with precision of `S`. The `value` can be a number or a string. The `S` (scale) parameter specifies the number of decimal places. diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 07266eb09a3..9387b442944 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -18,13 +18,13 @@ Each action is an operation on a column. The following actions are supported: -- [ADD COLUMN](#alter_add-column) — Adds a new column to the table. -- [DROP COLUMN](#alter_drop-column) — Deletes the column. -- [RENAME COLUMN](#alter_rename-column) — Renames an existing column. -- [CLEAR COLUMN](#alter_clear-column) — Resets column values. -- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. -- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL. -- [MODIFY COLUMN REMOVE](#modify-remove) — Removes one of the column properties. +- [ADD COLUMN](#add-column) — Adds a new column to the table. +- [DROP COLUMN](#drop-column) — Deletes the column. +- [RENAME COLUMN](#rename-column) — Renames an existing column. +- [CLEAR COLUMN](#clear-column) — Resets column values. +- [COMMENT COLUMN](#comment-column) — Adds a text comment to the column. +- [MODIFY COLUMN](#modify-column) — Changes column’s type, default expression and TTL. +- [MODIFY COLUMN REMOVE](#modify-column-remove) — Removes one of the column properties. - [MATERIALIZE COLUMN](#materialize-column) — Materializes the column in the parts where the column is missing. These actions are described in detail below. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 75c80add9b7..921ffb71066 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -7,18 +7,18 @@ sidebar_label: PARTITION The following operations with [partitions](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md) are available: -- [DETACH PARTITION](#alter_detach-partition) — Moves a partition to the `detached` directory and forget it. -- [DROP PARTITION](#alter_drop-partition) — Deletes a partition. -- [ATTACH PART\|PARTITION](#alter_attach-partition) — Adds a part or partition from the `detached` directory to the table. -- [ATTACH PARTITION FROM](#alter_attach-partition-from) — Copies the data partition from one table to another and adds. -- [REPLACE PARTITION](#alter_replace-partition) — Copies the data partition from one table to another and replaces. -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — Moves the data partition from one table to another. -- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition. -- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition. -- [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition. -- [UNFREEZE PARTITION](#alter_unfreeze-partition) — Removes a backup of a partition. -- [FETCH PARTITION\|PART](#alter_fetch-partition) — Downloads a part or partition from another server. -- [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume. +- [DETACH PARTITION\|PART](#detach-partitionpart) — Moves a partition or part to the `detached` directory and forget it. +- [DROP PARTITION\|PART](#drop-partitionpart) — Deletes a partition or part. +- [ATTACH PARTITION\|PART](#attach-partitionpart) — Adds a partition or part from the `detached` directory to the table. +- [ATTACH PARTITION FROM](#attach-partition-from) — Copies the data partition from one table to another and adds. +- [REPLACE PARTITION](#replace-partition) — Copies the data partition from one table to another and replaces. +- [MOVE PARTITION TO TABLE](#move-partition-to-table) — Moves the data partition from one table to another. +- [CLEAR COLUMN IN PARTITION](#clear-column-in-partition) — Resets the value of a specified column in a partition. +- [CLEAR INDEX IN PARTITION](#clear-index-in-partition) — Resets the specified secondary index in a partition. +- [FREEZE PARTITION](#freeze-partition) — Creates a backup of a partition. +- [UNFREEZE PARTITION](#unfreeze-partition) — Removes a backup of a partition. +- [FETCH PARTITION\|PART](#fetch-partitionpart) — Downloads a part or partition from another server. +- [MOVE PARTITION\|PART](#move-partitionpart) — Move partition/data part to another disk or volume. - [UPDATE IN PARTITION](#update-in-partition) — Update data inside the partition by condition. - [DELETE IN PARTITION](#delete-in-partition) — Delete data inside the partition by condition. diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index e02db6d4f6b..1d6edc5fa3d 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -312,11 +312,11 @@ The aggregation can be performed more effectively, if a table is sorted by some ### GROUP BY in External Memory You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`. -The [max_bytes_before_external_group_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. +The [max_bytes_before_external_group_by](../../../operations/settings/query-complexity.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. When using `max_bytes_before_external_group_by`, we recommend that you set `max_memory_usage` about twice as high. This is necessary because there are two stages to aggregation: reading the data and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn’t dumped, then stage 2 might require up to the same amount of memory as in stage 1. -For example, if [max_memory_usage](../../../operations/settings/settings.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and `max_memory_usage` to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. +For example, if [max_memory_usage](../../../operations/settings/query-complexity.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and `max_memory_usage` to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. With distributed query processing, external aggregation is performed on remote servers. In order for the requester server to use only a small amount of RAM, set `distributed_aggregation_memory_efficient` to 1. diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 33ddaa89435..60d95b17c4c 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -111,4 +111,3 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); - [The ‘MySQL’ table engine](../../engines/table-engines/integrations/mysql.md) - [Using MySQL as a source of external dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) -[Original article](https://clickhouse.com/docs/en/sql-reference/table_functions/mysql/) diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md index 1b99ec97553..0f4d22e90ce 100644 --- a/docs/ru/development/contrib.md +++ b/docs/ru/development/contrib.md @@ -18,6 +18,7 @@ sidebar_label: "Используемые сторонние библиотеки | aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) | | aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) | | aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) | +| base58 | [MIT](https://github.com/ClickHouse/base-x/blob/3e58874643c087f57e82b0ff03825c933fab945a/LICENSE) | | base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) | | boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) | | boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) | diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index 0857359e987..84f31c0afcc 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -183,7 +183,6 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 | - | - | |hadoop\_kerberos\_keytab | "" | |hadoop\_kerberos\_principal | "" | -|hadoop\_kerberos\_kinit\_command | kinit | ### Ограничения {#limitations} * `hadoop_security_kerberos_ticket_cache_path` и `libhdfs3_conf` могут быть определены только на глобальном, а не на пользовательском уровне @@ -196,7 +195,7 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 коммуникация с узлами данных не защищена SASL (`HADOOP_SECURE_DN_USER` надежный показатель такого подхода к безопасности). Используйте `tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh` для примера настроек. -Если `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` или `hadoop_kerberos_kinit_command` указаны в настройках, `kinit` будет вызван. `hadoop_kerberos_keytab` и `hadoop_kerberos_principal` обязательны в этом случае. Необходимо также будет установить `kinit` и файлы конфигурации krb5. +Если `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` или `hadoop_security_kerberos_ticket_cache_path` указаны в настройках, будет использоваться аутентификация с помощью Kerberos. `hadoop_kerberos_keytab` и `hadoop_kerberos_principal` обязательны в этом случае. ## Виртуальные столбцы {#virtual-columns} diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index b24a096015d..b51a0113302 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -167,7 +167,7 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format ### Поддержка Kerberos {#kafka-kerberos-support} Чтобы начать работу с Kafka с поддержкой Kerberos, добавьте дочерний элемент `security_protocol` со значением `sasl_plaintext`. Этого будет достаточно, если получен тикет на получение тикета (ticket-granting ticket) Kerberos и он кэшируется средствами ОС. -ClickHouse может поддерживать учетные данные Kerberos с помощью файла keytab. Рассмотрим дочерние элементы `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` и `sasl.kerberos.kinit.cmd`. +ClickHouse может поддерживать учетные данные Kerberos с помощью файла keytab. Рассмотрим дочерние элементы `sasl_kerberos_service_name`, `sasl_kerberos_keytab` и `sasl_kerberos_principal`. Пример: diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 6cd4cea403e..1acf5ec58b2 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -490,6 +490,48 @@ SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2); Возвращает сконвертированную из кодировки from в кодировку to строку s. +## Base58Encode(plaintext[, alphabet_name]), Base58Decode(plaintext[, alphabet_name]) {#base58} + +Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием указанного алфавита. + +**Синтаксис** + +```sql +base58Encode(decoded[, alphabet_name]) +base58Decode(encoded[, alphabet_name]) +``` + +**Аргументы** + +- `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). +- `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`. +- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `bitcoin`. + +**Возвращаемое значение** + +- Строка, содержащая раскодированный/закодированный первый аргумент. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример:** + +Запрос: + +``` sql +SELECT base58Encode('encode', 'flickr'); +SELECT base58Decode('izCFiDUY', 'ripple'); +``` + +Результат: +```text +┌─base58Encode('encode', 'flickr')─┐ +│ SvyTHb1D │ +└──────────────────────────────────┘ +┌─base58Decode('izCFiDUY', 'ripple')─┐ +│ decode │ +└────────────────────────────────────┘ +``` + ## base64Encode(s) {#base64encode} Производит кодирование строки s в base64-представление. diff --git a/docs/ru/sql-reference/statements/select/group-by.md b/docs/ru/sql-reference/statements/select/group-by.md index 5a306e63955..01df1b969bf 100644 --- a/docs/ru/sql-reference/statements/select/group-by.md +++ b/docs/ru/sql-reference/statements/select/group-by.md @@ -264,10 +264,10 @@ GROUP BY вычисляет для каждого встретившегося ### Группировка во внешней памяти {#select-group-by-in-external-memory} Можно включить сброс временных данных на диск, чтобы ограничить потребление оперативной памяти при выполнении `GROUP BY`. -Настройка [max_bytes_before_external_group_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. +Настройка [max_bytes_before_external_group_by](../../../operations/settings/query-complexity.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. При использовании `max_bytes_before_external_group_by`, рекомендуем выставить `max_memory_usage` приблизительно в два раза больше. Это следует сделать, потому что агрегация выполняется в две стадии: чтение и формирование промежуточных данных (1) и слияние промежуточных данных (2). Сброс данных на файловую систему может производиться только на стадии 1. Если сброса временных данных не было, то на стадии 2 может потребляться до такого же объёма памяти, как на стадии 1. -Например, если [max_memory_usage](../../../operations/settings/settings.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а `max_memory_usage` в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. +Например, если [max_memory_usage](../../../operations/settings/query-complexity.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а `max_memory_usage` в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. При распределённой обработке запроса внешняя агрегация производится на удалённых серверах. Для того чтобы на сервере-инициаторе запроса использовалось немного оперативки, нужно выставить настройку `distributed_aggregation_memory_efficient` в 1. diff --git a/docs/zh/engines/table-engines/mergetree-family/replication.md b/docs/zh/engines/table-engines/mergetree-family/replication.md index c3be3a382cb..69d1cd9d942 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replication.md +++ b/docs/zh/engines/table-engines/mergetree-family/replication.md @@ -18,11 +18,19 @@ 而 `CREATE`，`DROP`，`ATTACH`，`DETACH` 和 `RENAME` 语句只会在单个服务器上执行，不会被复制。 -- `The CREATE TABLE` 在运行此语句的服务器上创建一个新的可复制表。如果此表已存在其他服务器上，则给该表添加新副本。 -- `The DROP TABLE` 删除运行此查询的服务器上的副本。 -- `The RENAME` 重命名一个副本。换句话说，可复制表不同的副本可以有不同的名称。 +- `CREATE TABLE` 在运行此语句的服务器上创建一个新的可复制表。如果此表已存在其他服务器上，则给该表添加新副本。 +- `DROP TABLE` 删除运行此查询的服务器上的副本。 +- `RENAME` 重命名一个副本。换句话说，可复制表不同的副本可以有不同的名称。 -要使用副本，需在配置文件中设置 ZooKeeper 集群的地址。例如： +ClickHouse 使用 [Apache ZooKeeper](https://zookeeper.apache.org) 存储副本的元信息。请使用 ZooKeeper 3.4.5 或更高版本。 + +要使用副本，需在 [Zookeeper](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) 服务器的配置部分设置相应参数。 + +:::warning +不要忽视安全设置。 ClickHouse 支持 ZooKeeper 安全子系统中的 `digest` [ACL 方案](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) 。 +::: + +Zookeeper 集群地址的设置样例如下： ``` xml @@ -41,7 +49,41 @@ ``` -需要 ZooKeeper 3.4.5 或更高版本。 +通过以引擎参数的形式提供 ZooKeeper 集群的名称和路径，ClickHouse 同样支持将副本的元信息存储在备用 ZooKeeper 集群上。也就是说，支持将不同数据表的元数据存储在不同的 ZooKeeper 集群上。 + +设置备用 ZooKeeper 集群地址的样例如下： + +``` xml + + + + example_2_1 + 2181 + + + example_2_2 + 2181 + + + example_2_3 + 2181 + + + + + example_3_1 + 2181 + + + +``` + +为了将数据表的元数据存储到备用 ZooKeeper 集群而非默认 ZooKeeper 集群，我们可以通过如下 SQL 的方式创建使用 +ReplicatedMergeTree 引擎的数据表： + +``` +CREATE TABLE table_name ( ... ) ENGINE = ReplicatedMergeTree('zookeeper_name_configured_in_auxiliary_zookeepers:path', 'replica_name') ... +``` 你可以配置任何现有的 ZooKeeper 集群，系统会使用里面的目录来存取元数据（该目录在创建可复制表时指定）。 @@ -53,7 +95,9 @@ 对于非常大的集群，你可以把不同的 ZooKeeper 集群用于不同的分片。然而，即使 Yandex.Metrica 集群（大约300台服务器）也证明还不需要这么做。 -复制是多主异步。 `INSERT` 语句（以及 `ALTER` ）可以发给任意可用的服务器。数据会先插入到执行该语句的服务器上，然后被复制到其他服务器。由于它是异步的，在其他副本上最近插入的数据会有一些延迟。如果部分副本不可用，则数据在其可用时再写入。副本可用的情况下，则延迟时长是通过网络传输压缩数据块所需的时间。 +复制是多主异步。 `INSERT` 语句（以及 `ALTER` ）可以发给任意可用的服务器。数据会先插入到执行该语句的服务器上，然后被复制到其他服务器。由于它是异步的，在其他副本上最近插入的数据会有一些延迟。如果部分副本不可用，则数据在其可用时再写入。副本可用的情况下，则延迟时长是通过网络传输压缩数据块所需的时间。为复制表执行后台任务的线程数量，可以通过 [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) 进行设置。 + +`ReplicatedMergeTree` 引擎采用一个独立的线程池进行复制拉取。线程池的大小通过 [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) 进行限定，它可以在重启服务器时进行调整。 默认情况下，INSERT 语句仅等待一个副本写入成功后返回。如果数据只成功写入一个副本后该副本所在的服务器不再存在，则存储的数据会丢失。要启用数据写入多个副本才确认返回，使用 `insert_quorum` 选项。 @@ -75,6 +119,7 @@ - `zoo_path` — ZooKeeper 中该表的路径。 - `replica_name` — ZooKeeper 中的该表的副本名称。 +- `other_parameters` — 关于引擎的一系列参数，这个引擎即是用来创建复制的引擎，例如，`ReplacingMergeTree` 。 示例: @@ -90,7 +135,9 @@ ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) ``` -已弃用的建表语法示例： +
+ +已弃用的建表语法示例： ``` sql CREATE TABLE table_name @@ -101,17 +148,19 @@ CREATE TABLE table_name ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) ``` +
+ 如上例所示，这些参数可以包含宏替换的占位符，即大括号的部分。它们会被替换为配置文件里 ‘macros’ 那部分配置的值。示例： ``` xml 05 02 - example05-02-1.yandex.ru + example05-02-1 ``` -«ZooKeeper 中该表的路径»对每个可复制表都要是唯一的。不同分片上的表要有不同的路径。 +ZooKeeper 中该表的路径对每个可复制表都要是唯一的。不同分片上的表要有不同的路径。 这种情况下，路径包含下面这些部分： `/clickhouse/tables/` 是公共前缀，我们推荐使用这个。 @@ -119,7 +168,11 @@ CREATE TABLE table_name `{layer}-{shard}` 是分片标识部分。在此示例中，由于 Yandex.Metrica 集群使用了两级分片，所以它是由两部分组成的。但对于大多数情况来说，你只需保留 {shard} 占位符即可，它会替换展开为分片标识。 `table_name` 是该表在 ZooKeeper 中的名称。使其与 ClickHouse 中的表名相同比较好。 这里它被明确定义，跟 ClickHouse 表名不一样，它并不会被 RENAME 语句修改。 -*HINT*：你可以在前面添加一个数据库名称 `table_name` 也是 例如。 `db_name.table_name` +*HINT*：你也可以在 `table_name` 前面添加一个数据库名称。例如： `db_name.table_name` 。 + +两个内置的占位符 `{database}` 和 `{table}` 也可使用，它们可以展开成数据表名称和数据库名称（只有当这些宏指令在 `macros` 部分已经定义时才可以）。因此 ZooKeeper 路径可以指定为 `'/clickhouse/tables/{layer}-{shard}/{database}/{table}'` 。 + +当使用这些内置占位符时，请当心数据表重命名。 ZooKeeper 中的路径无法变更，而当数据表被重命名时，宏命令将展开为另一个路径，数据表将指向一个 ZooKeeper 上并不存在的路径，并因此转变为只读模式。 副本名称用于标识同一个表分片的不同副本。你可以使用服务器名称，如上例所示。同个分片中不同副本的副本名称要唯一。 @@ -127,6 +180,31 @@ CREATE TABLE table_name 使用大型集群时，我们建议使用宏替换，因为它可以降低出错的可能性。 +你可以在服务器的配置文件中指定 `Replicated` 数据表引擎的默认参数。例如： + +```xml +/clickhouse/tables/{shard}/{database}/{table} +{replica} +``` + +这样，你可以在建表时省略参数： + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree +ORDER BY x; +``` + +它等价于： + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}') +ORDER BY x; +``` + 在每个副本服务器上运行 `CREATE TABLE` 查询。将创建新的复制表，或给现有表添加新副本。 如果其他副本上已包含了某些数据，在表上添加新副本，则在运行语句后，数据会从其他副本复制到新副本。换句话说，新副本会与其他副本同步。 @@ -164,7 +242,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data 1. 在服务器上安装 ClickHouse。在包含分片标识符和副本的配置文件中正确定义宏配置，如果有用到的话， 2. 如果服务器上有非复制表则必须手动复制，可以从副本服务器上（在 `/var/lib/clickhouse/data/db_name/table_name/` 目录中）复制它们的数据。 3. 从副本服务器上中复制位于 `/var/lib/clickhouse/metadata/` 中的表定义信息。如果在表定义信息中显式指定了分片或副本标识符，请更正它以使其对应于该副本。（另外，启动服务器，然后会在 `/var/lib/clickhouse/metadata/` 中的.sql文件中生成所有的 `ATTACH TABLE` 语句。） - 4.要开始恢复，ZooKeeper 中创建节点 `/path_to_table/replica_name/flags/force_restore_data`，节点内容不限，或运行命令来恢复所有复制的表：`sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` +4. 要开始恢复，ZooKeeper 中创建节点 `/path_to_table/replica_name/flags/force_restore_data`，节点内容不限，或运行命令来恢复所有复制的表：`sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` 然后启动服务器（如果它已运行则重启）。数据会从副本中下载。 @@ -199,4 +277,12 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data 如果 ZooKeeper 中的数据丢失或损坏，如上所述，你可以通过将数据转移到非复制表来保存数据。 -[来源文章](https://clickhouse.com/docs/en/operations/table_engines/replication/) +**参考** + +- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) +- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) + +[原始文章](https://clickhouse.com/docs/en/operations/table_engines/replication/) diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index 3d66f79cb46..5fa77bffa78 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -16,7 +16,14 @@ sidebar_label: 使用教程 例如，您选择`deb`安装包，执行: ``` bash -{% include 'install/deb.sh' %} +sudo apt-get install -y apt-transport-https ca-certificates dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 + +echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ + /etc/apt/sources.list.d/clickhouse.list +sudo apt-get update + +sudo apt-get install -y clickhouse-server clickhouse-client ``` 在我们安装的软件中包含这些包: diff --git a/docs/zh/sql-reference/statements/select/group-by.md b/docs/zh/sql-reference/statements/select/group-by.md index bebe4b164bf..33fa27b8dd9 100644 --- a/docs/zh/sql-reference/statements/select/group-by.md +++ b/docs/zh/sql-reference/statements/select/group-by.md @@ -116,11 +116,11 @@ GROUP BY domain ### 在外部存储器中分组 {#select-group-by-in-external-memory} 您可以启用将临时数据转储到磁盘以限制内存使用期间 `GROUP BY`. -该 [max_bytes_before_external_group_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) 设置确定倾销的阈值RAM消耗 `GROUP BY` 临时数据到文件系统。 如果设置为0（默认值），它将被禁用。 +该 [max_bytes_before_external_group_by](../../../operations/settings/query-complexity.md#settings-max_bytes_before_external_group_by) 设置确定倾销的阈值RAM消耗 `GROUP BY` 临时数据到文件系统。 如果设置为0（默认值），它将被禁用。 使用时 `max_bytes_before_external_group_by`，我们建议您设置 `max_memory_usage` 大约两倍高。 这是必要的，因为聚合有两个阶段：读取数据和形成中间数据（1）和合并中间数据（2）。 将数据转储到文件系统只能在阶段1中发生。 如果未转储临时数据，则阶段2可能需要与阶段1相同的内存量。 -例如，如果 [max_memory_usage](../../../operations/settings/settings.md#settings_max_memory_usage) 设置为10000000000，你想使用外部聚合，这是有意义的设置 `max_bytes_before_external_group_by` 到10000000000，和 `max_memory_usage` 到20000000000。 当触发外部聚合（如果至少有一个临时数据转储）时，RAM的最大消耗仅略高于 `max_bytes_before_external_group_by`. +例如，如果 [max_memory_usage](../../../operations/settings/query-complexity.md#settings_max_memory_usage) 设置为10000000000，你想使用外部聚合，这是有意义的设置 `max_bytes_before_external_group_by` 到10000000000，和 `max_memory_usage` 到20000000000。 当触发外部聚合（如果至少有一个临时数据转储）时，RAM的最大消耗仅略高于 `max_bytes_before_external_group_by`. 通过分布式查询处理，在远程服务器上执行外部聚合。 为了使请求者服务器只使用少量的RAM，设置 `distributed_aggregation_memory_efficient` 到1。 diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a2c6eb1a27e..2282daac90b 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -2,7 +2,7 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () -include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) +include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) # The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), # each of them may be built and linked as a separate library. @@ -18,6 +18,12 @@ option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_A option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)" ${ENABLE_CLICKHOUSE_ALL}) +if (CLICKHOUSE_SPLIT_BINARY OR NOT ENABLE_UTILS) + option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF) +else () + option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON) +endif () + # https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/ option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL}) @@ -101,6 +107,12 @@ else() message(STATUS "Local mode: OFF") endif() +if (ENABLE_CLICKHOUSE_SELF_EXTRACTING) + message(STATUS "Self-extracting executable: ON") +else() + message(STATUS "Self-extracting executable: OFF") +endif() + if (ENABLE_CLICKHOUSE_BENCHMARK) message(STATUS "Benchmark mode: ON") else() @@ -266,6 +278,10 @@ if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE) add_subdirectory (library-bridge) endif () +if (ENABLE_CLICKHOUSE_SELF_EXTRACTING) + add_subdirectory (self-extracting) +endif () + if (CLICKHOUSE_ONE_SHARED) add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} @@ -511,10 +527,10 @@ else () add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM) endif() - if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${STRIPPED_BINARIES_OUTPUT} BINARY_PATH clickhouse) + if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR}) install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() endif() diff --git a/programs/disks/CommandList.cpp b/programs/disks/CommandList.cpp index 8cf353aa64c..8c6bfac3a9b 100644 --- a/programs/disks/CommandList.cpp +++ b/programs/disks/CommandList.cpp @@ -16,13 +16,21 @@ public: CommandList() { command_name = "list"; + command_option_description.emplace(createOptionsDescription("Allowed options", getTerminalWidth())); description = "List files (the default disk is used by default)\nPath should be in format './' or './path' or 'path'"; usage = "list [OPTION]... ..."; + command_option_description->add_options() + ("recursive", "recursively list all directories") + ; } void processOptions( - Poco::Util::LayeredConfiguration &, - po::variables_map &) const override{} + Poco::Util::LayeredConfiguration & config, + po::variables_map & options) const override + { + if (options.count("recursive")) + config.setBool("recursive", true); + } void execute( const std::vector & command_arguments, @@ -39,16 +47,45 @@ public: String path = command_arguments[0]; - std::vector file_names; DiskPtr disk = global_context->getDisk(disk_name); String full_path = fullPathWithValidate(disk, path); + bool recursive = config.getBool("recursive", false); + + if (recursive) + listRecursive(disk, full_path); + else + list(disk, full_path); + } + +private: + static void list(const DiskPtr & disk, const std::string & full_path) + { + std::vector file_names; disk->listFiles(full_path, file_names); for (const auto & file_name : file_names) std::cout << file_name << '\n'; } + + static void listRecursive(const DiskPtr & disk, const std::string & full_path) + { + std::vector file_names; + disk->listFiles(full_path, file_names); + + std::cout << full_path << ":\n"; + for (const auto & file_name : file_names) + std::cout << file_name << '\n'; + std::cout << "\n"; + + for (const auto & file_name : file_names) + { + auto path = full_path + "/" + file_name; + if (disk->isDirectory(path)) + listRecursive(disk, path); + } + } }; } diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index c77b335b615..cf6c8a6e975 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -131,10 +131,10 @@ if (BUILD_STANDALONE_KEEPER) add_dependencies(clickhouse-keeper clickhouse_keeper_configs) set_target_properties(clickhouse-keeper PROPERTIES RUNTIME_OUTPUT_DIRECTORY ../) - if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-keeper) + if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-keeper) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) install(TARGETS clickhouse-keeper RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() endif() diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 90ce3d8be7f..a80f2568f04 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -1,4 +1,4 @@ -include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) +include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES library-bridge.cpp @@ -24,9 +24,9 @@ target_link_libraries(clickhouse-library-bridge PRIVATE set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-library-bridge) +if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-library-bridge) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index b530e08ca26..f64bec9892f 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -1,4 +1,4 @@ -include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) +include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_ODBC_BRIDGE_SOURCES ColumnInfoHandler.cpp @@ -39,10 +39,10 @@ if (USE_GDB_ADD_INDEX) add_custom_command(TARGET clickhouse-odbc-bridge POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} ../clickhouse-odbc-bridge COMMENT "Adding .gdb-index to clickhouse-odbc-bridge" VERBATIM) endif() -if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-odbc-bridge) +if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-odbc-bridge) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() diff --git a/programs/odbc-bridge/ODBCPooledConnectionFactory.h b/programs/odbc-bridge/ODBCPooledConnectionFactory.h index 4d8d3f50ab9..f6185bffd1d 100644 --- a/programs/odbc-bridge/ODBCPooledConnectionFactory.h +++ b/programs/odbc-bridge/ODBCPooledConnectionFactory.h @@ -91,7 +91,11 @@ T execute(nanodbc::ConnectionHolderPtr connection_holder, std::function & /*args*/) /// We load temporary database first, because projections need it. database_catalog.initializeAndLoadTemporaryDatabase(); loadMetadataSystem(global_context); + maybeConvertOrdinaryDatabaseToAtomic(global_context, DatabaseCatalog::instance().getSystemDatabase()); /// After attaching system databases we can initialize system log. global_context->initializeSystemLogs(); global_context->setSystemZooKeeperLogAfterInitializationIfNeeded(); diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml index 2b6c4d9f770..0f11efab8a3 100644 --- a/programs/server/embedded.xml +++ b/programs/server/embedded.xml @@ -12,7 +12,6 @@ ./ - 8589934592 true diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 0322ad7457b..bd1344a6f14 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ namespace DB namespace ErrorCodes { + extern const int CANNOT_RESTORE_TABLE; extern const int LOGICAL_ERROR; } @@ -139,7 +141,7 @@ namespace } catch (Exception & e) { - e.addMessage("While parsing " + file_path); + e.addMessage("While parsing " + file_path + " from backup"); throw; } } @@ -225,7 +227,7 @@ namespace } } - AccessRightsElements getRequiredAccessToRestore(const std::unordered_map & entities) + AccessRightsElements getRequiredAccessToRestore(const std::vector> & entities) { AccessRightsElements res; for (const auto & entity : entities | boost::adaptors::map_values) @@ -294,65 +296,78 @@ namespace } } -void backupAccessEntities( - BackupEntriesCollector & backup_entries_collector, + +std::pair makeBackupEntryForAccess( + const std::vector> access_entities, const String & data_path_in_backup, - const AccessControl & access_control, - AccessEntityType type) + size_t counter, + const AccessControl & access_control) { - auto entities = access_control.readAllForBackup(type, backup_entries_collector.getBackupSettings()); - auto dependencies = readDependenciesNamesAndTypes(findDependencies(entities), access_control); + auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control); AccessEntitiesInBackup ab; - boost::range::copy(entities, std::inserter(ab.entities, ab.entities.end())); + boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end())); ab.dependencies = std::move(dependencies); - backup_entries_collector.addBackupEntry(fs::path{data_path_in_backup} / "access.txt", ab.toBackupEntry()); + String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ... + String file_path_in_backup = fs::path{data_path_in_backup} / filename; + return {file_path_in_backup, ab.toBackupEntry()}; } -AccessRestoreTask::AccessRestoreTask( - const BackupPtr & backup_, const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_) - : backup(backup_), restore_settings(restore_settings_), restore_coordination(restore_coordination_) +AccessRestorerFromBackup::AccessRestorerFromBackup( + const BackupPtr & backup_, const RestoreSettings & restore_settings_) + : backup(backup_), allow_unresolved_access_dependencies(restore_settings_.allow_unresolved_access_dependencies) { } -AccessRestoreTask::~AccessRestoreTask() = default; +AccessRestorerFromBackup::~AccessRestorerFromBackup() = default; -void AccessRestoreTask::addDataPath(const String & data_path) +void AccessRestorerFromBackup::addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs) { if (!data_paths.emplace(data_path).second) return; - String file_path = fs::path{data_path} / "access.txt"; - auto backup_entry = backup->readFile(file_path); - auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, file_path); + fs::path data_path_in_backup_fs = data_path; + Strings filenames = backup->listFiles(data_path); + if (filenames.empty()) + return; + + for (const String & filename : filenames) + { + if (!filename.starts_with("access") || !filename.ends_with(".txt")) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File name {} doesn't match the wildcard \"access*.txt\"", + table_name_for_logs.getFullName(), String{data_path_in_backup_fs / filename}); + } + + ::sort(filenames.begin(), filenames.end()); + + for (const String & filename : filenames) + { + String filepath_in_backup = data_path_in_backup_fs / filename; + auto backup_entry = backup->readFile(filepath_in_backup); + auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, filepath_in_backup); + + boost::range::copy(ab.entities, std::back_inserter(entities)); + boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); + } - boost::range::copy(ab.entities, std::inserter(entities, entities.end())); - boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); for (const auto & id : entities | boost::adaptors::map_keys) dependencies.erase(id); } -bool AccessRestoreTask::hasDataPath(const String & data_path) const -{ - return data_paths.contains(data_path); -} - -AccessRightsElements AccessRestoreTask::getRequiredAccess() const +AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const { return getRequiredAccessToRestore(entities); } -void AccessRestoreTask::restore(AccessControl & access_control) const +std::vector> AccessRestorerFromBackup::getAccessEntities(const AccessControl & access_control) const { - auto old_to_new_ids = resolveDependencies(dependencies, access_control, restore_settings.allow_unresolved_access_dependencies); + auto new_entities = entities; - std::vector> new_entities; - boost::range::copy(entities, std::back_inserter(new_entities)); + auto old_to_new_ids = resolveDependencies(dependencies, access_control, allow_unresolved_access_dependencies); generateRandomIDs(new_entities, old_to_new_ids); - replaceDependencies(new_entities, old_to_new_ids); - access_control.insertFromBackup(new_entities, restore_settings, restore_coordination); + return new_entities; } } diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index 5c70e268eae..74f889e2c00 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -9,46 +9,45 @@ namespace DB { class AccessControl; enum class AccessEntityType; -class BackupEntriesCollector; -class RestorerFromBackup; -class IBackup; -using BackupPtr = std::shared_ptr; -class IRestoreCoordination; struct IAccessEntity; using AccessEntityPtr = std::shared_ptr; class AccessRightsElements; +class IBackup; +using BackupPtr = std::shared_ptr; +class IBackupEntry; +using BackupEntryPtr = std::shared_ptr; +struct RestoreSettings; +struct QualifiedTableName; /// Makes a backup of access entities of a specified type. -void backupAccessEntities( - BackupEntriesCollector & backup_entries_collector, +std::pair makeBackupEntryForAccess( + const std::vector> access_entities, const String & data_path_in_backup, - const AccessControl & access_control, - AccessEntityType type); + size_t counter, + const AccessControl & access_control); + /// Restores access entities from a backup. -class AccessRestoreTask +class AccessRestorerFromBackup { public: - AccessRestoreTask( - const BackupPtr & backup_, const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_); - ~AccessRestoreTask(); + AccessRestorerFromBackup(const BackupPtr & backup_, const RestoreSettings & restore_settings_); + ~AccessRestorerFromBackup(); /// Adds a data path to loads access entities from. - void addDataPath(const String & data_path); - bool hasDataPath(const String & data_path) const; + void addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs); /// Checks that the current user can do restoring. AccessRightsElements getRequiredAccess() const; /// Inserts all access entities loaded from all the paths added by addDataPath(). - void restore(AccessControl & access_control) const; + std::vector> getAccessEntities(const AccessControl & access_control) const; private: BackupPtr backup; - RestoreSettings restore_settings; - std::shared_ptr restore_coordination; - std::unordered_map entities; + bool allow_unresolved_access_dependencies = false; + std::vector> entities; std::unordered_map> dependencies; std::unordered_set data_paths; }; diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index aa58044a6b0..7152820b5bc 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -459,20 +459,9 @@ UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Ne } } -void AccessControl::backup(BackupEntriesCollector & backup_entries_collector, AccessEntityType type, const String & data_path_in_backup) const +void AccessControl::restoreFromBackup(RestorerFromBackup & restorer) { - backupAccessEntities(backup_entries_collector, data_path_in_backup, *this, type); -} - -void AccessControl::restore(RestorerFromBackup & restorer, const String & data_path_in_backup) -{ - /// The restorer must already know about `data_path_in_backup`, but let's check. - restorer.checkPathInBackupToRestoreAccess(data_path_in_backup); -} - -void AccessControl::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) -{ - MultipleAccessStorage::insertFromBackup(entities_from_backup, restore_settings, restore_coordination); + MultipleAccessStorage::restoreFromBackup(restorer); changes_notifier->sendNotifications(); } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 90ad2895122..22ff0a488f7 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -42,8 +42,6 @@ class ClientInfo; class ExternalAuthenticators; class AccessChangesNotifier; struct Settings; -class BackupEntriesCollector; -class RestorerFromBackup; /// Manages access control entities. @@ -121,8 +119,7 @@ public: UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const; /// Makes a backup of access entities. - void backup(BackupEntriesCollector & backup_entries_collector, AccessEntityType type, const String & data_path_in_backup) const; - static void restore(RestorerFromBackup & restorer, const String & data_path_in_backup); + void restoreFromBackup(RestorerFromBackup & restorer) override; void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); @@ -198,8 +195,6 @@ public: /// Gets manager of notifications. AccessChangesNotifier & getChangesNotifier(); - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; - private: class ContextAccessCache; class CustomSettingsPrefixes; diff --git a/src/Access/CMakeLists.txt b/src/Access/CMakeLists.txt index e69de29bb2d..83bbe418246 100644 --- a/src/Access/CMakeLists.txt +++ b/src/Access/CMakeLists.txt @@ -0,0 +1,3 @@ +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 41739406de4..38e006f18e7 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -25,6 +25,7 @@ enum class AccessType M(SHOW_DICTIONARIES, "", DICTIONARY, SHOW) /* allows to execute SHOW DICTIONARIES, SHOW CREATE DICTIONARY, EXISTS ; implicitly enabled by any grant on the dictionary */\ M(SHOW, "", GROUP, ALL) /* allows to execute SHOW, USE, EXISTS, CHECK, DESCRIBE */\ + M(SHOW_CACHES, "", GROUP, ALL) \ \ M(SELECT, "", COLUMN, ALL) \ M(INSERT, "", COLUMN, ALL) \ @@ -138,6 +139,7 @@ enum class AccessType M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_MMAP_CACHE, "SYSTEM DROP MMAP, DROP MMAP CACHE, DROP MMAP", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_COMPILED_EXPRESSION_CACHE, "SYSTEM DROP COMPILED EXPRESSION, DROP COMPILED EXPRESSION CACHE, DROP COMPILED EXPRESSIONS", GLOBAL, SYSTEM_DROP_CACHE) \ + M(SYSTEM_DROP_FILESYSTEM_CACHE, "SYSTEM DROP FILESYSTEM CACHE, DROP FILESYSTEM CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_CACHE, "DROP CACHE", GROUP, SYSTEM) \ M(SYSTEM_RELOAD_CONFIG, "RELOAD CONFIG", GLOBAL, SYSTEM_RELOAD) \ M(SYSTEM_RELOAD_SYMBOLS, "RELOAD SYMBOLS", GLOBAL, SYSTEM_RELOAD) \ diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index a9b7a6a265b..994abc7b53a 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -153,15 +154,7 @@ namespace bool tryParseUUID(const String & str, UUID & id) { - try - { - id = parseFromString(str); - return true; - } - catch (...) - { - return false; - } + return tryParse(id, str); } } @@ -658,19 +651,24 @@ void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const } -void DiskAccessStorage::insertFromBackup( - const std::vector> & entities_from_backup, - const RestoreSettings & restore_settings, - std::shared_ptr) +void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; - for (const auto & [id, entity] : entities_from_backup) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] + { + for (const auto & [id, entity] : entities) + insertWithID(id, entity, replace_if_exists, throw_if_exists); + }); } } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 1bdefbf82f9..d3bd61ff353 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -30,7 +30,7 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void restoreFromBackup(RestorerFromBackup & restorer) override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 476b1674ce1..fa9c78816c7 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -2,9 +2,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -520,26 +523,30 @@ bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddr } -bool IAccessStorage::isRestoreAllowed() const -{ - return isBackupAllowed() && !isReadOnly(); -} - -std::vector> IAccessStorage::readAllForBackup(AccessEntityType type, const BackupSettings &) const +void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const { if (!isBackupAllowed()) throwBackupNotAllowed(); - auto res = readAllWithIDs(type); - boost::range::remove_erase_if(res, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); - return res; + auto entities = readAllWithIDs(type); + boost::range::remove_erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); + + auto backup_entry = makeBackupEntryForAccess( + entities, + data_path_in_backup, + backup_entries_collector.getAccessCounter(type), + backup_entries_collector.getContext()->getAccessControl()); + + backup_entries_collector.addBackupEntry(backup_entry); } -void IAccessStorage::insertFromBackup(const std::vector> &, const RestoreSettings &, std::shared_ptr) + +void IAccessStorage::restoreFromBackup(RestorerFromBackup &) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "insertFromBackup() is not implemented in {}", getStorageType()); + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "restoreFromBackup() is not implemented in {}", getStorageType()); } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 7b43309204d..394d3ed6358 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -18,9 +18,8 @@ struct User; class Credentials; class ExternalAuthenticators; enum class AuthenticationType; -struct BackupSettings; -struct RestoreSettings; -class IRestoreCoordination; +class BackupEntriesCollector; +class RestorerFromBackup; /// Contains entities, i.e. instances of classes derived from IAccessEntity. /// The implementations of this class MUST be thread-safe. @@ -158,11 +157,11 @@ public: /// Returns true if this storage can be stored to or restored from a backup. virtual bool isBackupAllowed() const { return false; } - virtual bool isRestoreAllowed() const; + virtual bool isRestoreAllowed() const { return isBackupAllowed() && !isReadOnly(); } /// Makes a backup of this access storage. - virtual std::vector> readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const; - virtual void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination); + virtual void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const; + virtual void restoreFromBackup(RestorerFromBackup & restorer); protected: virtual std::optional findImpl(AccessEntityType type, const String & name) const = 0; diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp new file mode 100644 index 00000000000..1bba82bc46f --- /dev/null +++ b/src/Access/KerberosInit.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#if USE_KRB5 +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int KERBEROS_ERROR; +} +} + +namespace +{ +struct K5Data +{ + krb5_context ctx; + krb5_ccache out_cc; + krb5_principal me; + char * name; + krb5_boolean switch_to_cache; +}; + +/** + * This class implements programmatic implementation of kinit. + */ +class KerberosInit : boost::noncopyable +{ +public: + void init(const String & keytab_file, const String & principal, const String & cache_name = ""); + ~KerberosInit(); +private: + struct K5Data k5 {}; + krb5_ccache defcache = nullptr; + krb5_get_init_creds_opt * options = nullptr; + // Credentials structure including ticket, session key, and lifetime info. + krb5_creds my_creds; + krb5_keytab keytab = nullptr; + krb5_principal defcache_princ = nullptr; + String fmtError(krb5_error_code code) const; +}; +} + + +String KerberosInit::fmtError(krb5_error_code code) const +{ + const char *msg; + msg = krb5_get_error_message(k5.ctx, code); + String fmt_error = fmt::format(" ({}, {})", code, msg); + krb5_free_error_message(k5.ctx, msg); + return fmt_error; +} + +void KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) +{ + auto * log = &Poco::Logger::get("KerberosInit"); + LOG_TRACE(log,"Trying to authenticate with Kerberos v5"); + + krb5_error_code ret; + + const char *deftype = nullptr; + + if (!std::filesystem::exists(keytab_file)) + throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); + + ret = krb5_init_context(&k5.ctx); + if (ret) + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while initializing Kerberos 5 library ({})", ret); + + if (!cache_name.empty()) + { + ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); + if (ret) + throw Exception("Error in resolving cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + LOG_TRACE(log,"Resolved cache"); + } + else + { + // Resolve the default cache and get its type and default principal (if it is initialized). + ret = krb5_cc_default(k5.ctx, &defcache); + if (ret) + throw Exception("Error while getting default cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + LOG_TRACE(log,"Resolved default cache"); + deftype = krb5_cc_get_type(k5.ctx, defcache); + if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) + defcache_princ = nullptr; + } + + // Use the specified principal name. + ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me); + if (ret) + throw Exception("Error when parsing principal name " + principal + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + + // Cache related commands + if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype)) + { + // Use an existing cache for the client principal if we can. + ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc); + if (ret && ret != KRB5_CC_NOTFOUND) + throw Exception("Error while searching for cache for " + principal + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + if (0 == ret) + { + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + k5.switch_to_cache = 1; + } + else if (defcache_princ != nullptr) + { + // Create a new cache to avoid overwriting the initialized default cache. + ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); + if (ret) + throw Exception("Error while generating new cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + k5.switch_to_cache = 1; + } + } + + // Use the default cache if we haven't picked one yet. + if (k5.out_cc == nullptr) + { + k5.out_cc = defcache; + defcache = nullptr; + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + } + + ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); + if (ret) + throw Exception("Error when unparsing name" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + LOG_TRACE(log,"Using principal: {}", k5.name); + + // Allocate a new initial credential options structure. + ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); + if (ret) + throw Exception("Error in options allocation" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + + // Resolve keytab + ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); + if (ret) + throw Exception("Error in resolving keytab "+keytab_file + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + LOG_TRACE(log,"Using keytab: {}", keytab_file); + + // Set an output credential cache in initial credential options. + ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); + if (ret) + throw Exception("Error in setting output credential cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + + // Action: init or renew + LOG_TRACE(log,"Trying to renew credentials"); + memset(&my_creds, 0, sizeof(my_creds)); + // Get renewed credential from KDC using an existing credential from output cache. + ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); + if (ret) + { + LOG_TRACE(log,"Renew failed {}", fmtError(ret)); + LOG_TRACE(log,"Trying to get initial credentials"); + // Request KDC for an initial credentials using keytab. + ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); + if (ret) + throw Exception("Error in getting initial credentials" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + else + LOG_TRACE(log,"Got initial credentials"); + } + else + { + LOG_TRACE(log,"Successful renewal"); + // Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal. + ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); + if (ret) + throw Exception("Error when initializing cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + LOG_TRACE(log,"Initialized cache"); + // Store credentials in a credential cache. + ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); + if (ret) + LOG_TRACE(log,"Error while storing credentials"); + LOG_TRACE(log,"Stored credentials"); + } + + if (k5.switch_to_cache) + { + // Make a credential cache the primary cache for its collection. + ret = krb5_cc_switch(k5.ctx, k5.out_cc); + if (ret) + throw Exception("Error while switching to new cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + } + + LOG_TRACE(log,"Authenticated to Kerberos v5"); +} + +KerberosInit::~KerberosInit() +{ + if (k5.ctx) + { + if (defcache) + krb5_cc_close(k5.ctx, defcache); + krb5_free_principal(k5.ctx, defcache_princ); + + if (options) + krb5_get_init_creds_opt_free(k5.ctx, options); + if (my_creds.client == k5.me) + my_creds.client = nullptr; + krb5_free_cred_contents(k5.ctx, &my_creds); + if (keytab) + krb5_kt_close(k5.ctx, keytab); + + krb5_free_unparsed_name(k5.ctx, k5.name); + krb5_free_principal(k5.ctx, k5.me); + if (k5.out_cc != nullptr) + krb5_cc_close(k5.ctx, k5.out_cc); + krb5_free_context(k5.ctx); + } +} + +void kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) +{ + // Using mutex to prevent cache file corruptions + static std::mutex kinit_mtx; + std::lock_guard lck(kinit_mtx); + KerberosInit k_init; + k_init.init(keytab_file, principal, cache_name); +} +#endif // USE_KRB5 diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h new file mode 100644 index 00000000000..5a11a275529 --- /dev/null +++ b/src/Access/KerberosInit.h @@ -0,0 +1,11 @@ +#pragma once + +#include "config_core.h" + +#include + +#if USE_KRB5 + +void kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); + +#endif // USE_KRB5 diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index ad877e263ad..60669532e25 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -272,19 +273,24 @@ void MemoryAccessStorage::setAll(const std::vector> & entities_from_backup, - const RestoreSettings & restore_settings, - std::shared_ptr) +void MemoryAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; - for (const auto & [id, entity] : entities_from_backup) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] + { + for (const auto & [id, entity] : entities) + insertWithID(id, entity, replace_if_exists, throw_if_exists); + }); } } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index aa4cd08252c..5c8d33ed443 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -29,7 +29,7 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void restoreFromBackup(RestorerFromBackup & restorer) override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 6f654f68e57..e7151cc7b4b 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -383,40 +383,38 @@ bool MultipleAccessStorage::isRestoreAllowed() const } -std::vector> MultipleAccessStorage::readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const +void MultipleAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const { - std::vector> res; auto storages = getStoragesInternal(); - size_t count = 0; + bool allowed = false; for (const auto & storage : *storages) { if (storage->isBackupAllowed()) { - insertAtEnd(res, storage->readAllForBackup(type, backup_settings)); - ++count; + storage->backup(backup_entries_collector, data_path_in_backup, type); + allowed = true; } } - if (!count) + if (!allowed) throwBackupNotAllowed(); - - return res; } - -void MultipleAccessStorage::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) +void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { auto storages = getStoragesInternal(); + for (const auto & storage : *storages) { if (storage->isRestoreAllowed()) { - storage->insertFromBackup(entities_from_backup, restore_settings, restore_coordination); + storage->restoreFromBackup(restorer); return; } } - throwRestoreNotAllowed(); + + throwBackupNotAllowed(); } } diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 2eacdafd3f3..58cf09fd0ff 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -45,8 +45,8 @@ public: bool isBackupAllowed() const override; bool isRestoreAllowed() const override; - std::vector> readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const override; - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; + void restoreFromBackup(RestorerFromBackup & restorer) override; protected: std::optional findImpl(AccessEntityType type, const String & name) const override; diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 6a9d716c2f9..f6c8d0a7153 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -2,10 +2,14 @@ #include #include #include +#include +#include +#include #include +#include #include #include -#include +#include #include #include #include @@ -13,6 +17,7 @@ #include #include #include +#include namespace DB @@ -613,19 +618,64 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if return entry.entity; } -void ReplicatedAccessStorage::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) + +void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const +{ + if (!isBackupAllowed()) + throwBackupNotAllowed(); + + auto entities = readAllWithIDs(type); + boost::range::remove_erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); + + auto backup_entry_with_path = makeBackupEntryForAccess( + entities, + data_path_in_backup, + backup_entries_collector.getAccessCounter(type), + backup_entries_collector.getContext()->getAccessControl()); + + auto backup_coordination = backup_entries_collector.getBackupCoordination(); + backup_coordination->addReplicatedAccessPath(zookeeper_path, backup_entry_with_path.first); + String current_host_id = backup_entries_collector.getBackupSettings().host_id; + backup_coordination->setReplicatedAccessHost(zookeeper_path, current_host_id); + + backup_entries_collector.addPostTask( + [backup_entry = backup_entry_with_path.second, + zookeeper_path = zookeeper_path, + current_host_id, + &backup_entries_collector, + backup_coordination] + { + if (current_host_id != backup_coordination->getReplicatedAccessHost(zookeeper_path)) + return; + + for (const String & path : backup_coordination->getReplicatedAccessPaths(zookeeper_path)) + backup_entries_collector.addBackupEntry(path, backup_entry); + }); +} + + +void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); + auto restore_coordination = restorer.getRestoreCoordination(); if (!restore_coordination->acquireReplicatedAccessStorage(zookeeper_path)) return; - bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; - for (const auto & [id, entity] : entities_from_backup) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] + { + for (const auto & [id, entity] : entities) + insertWithID(id, entity, replace_if_exists, throw_if_exists); + }); } } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 7cccdc1793f..6311e2ac7c0 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -38,7 +38,8 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; + void restoreFromBackup(RestorerFromBackup & restorer) override; private: String zookeeper_path; diff --git a/src/Access/examples/CMakeLists.txt b/src/Access/examples/CMakeLists.txt new file mode 100644 index 00000000000..07f75ca0b47 --- /dev/null +++ b/src/Access/examples/CMakeLists.txt @@ -0,0 +1,4 @@ +if (TARGET ch_contrib::krb5) + add_executable (kerberos_init kerberos_init.cpp) + target_link_libraries (kerberos_init PRIVATE dbms ch_contrib::krb5) +endif() diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp new file mode 100644 index 00000000000..5dbe92a5b57 --- /dev/null +++ b/src/Access/examples/kerberos_init.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include + +/** The example demonstrates using of kerberosInit function to obtain and cache Kerberos ticket-granting ticket. + * The first argument specifies keytab file. The second argument specifies principal name. + * The third argument is optional. It specifies credentials cache location. + * After successful run of kerberos_init it is useful to call klist command to list cached Kerberos tickets. + * It is also useful to run kdestroy to destroy Kerberos tickets if needed. + */ + +using namespace DB; + +int main(int argc, char ** argv) +{ + std::cout << "Kerberos Init" << "\n"; + + if (argc < 3) + { + std::cout << "kerberos_init obtains and caches an initial ticket-granting ticket for principal." << "\n\n"; + std::cout << "Usage:" << "\n" << " kerberos_init keytab principal [cache]" << "\n"; + return 0; + } + + String cache_name = ""; + if (argc == 4) + cache_name = argv[3]; + + Poco::AutoPtr app_channel(new Poco::ConsoleChannel(std::cerr)); + Poco::Logger::root().setChannel(app_channel); + Poco::Logger::root().setLevel("trace"); + + try + { + kerberosInit(argv[1], argv[2], cache_name); + } + catch (const Exception & e) + { + std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + return -1; + } + std::cout << "Done" << "\n"; + return 0; +} diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h index 77660c54d32..6e140f4b9cf 100644 --- a/src/AggregateFunctions/Helpers.h +++ b/src/AggregateFunctions/Helpers.h @@ -3,34 +3,6 @@ #include #include -#define FOR_BASIC_NUMERIC_TYPES(M) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(Int8) \ - M(Int16) \ - M(Int32) \ - M(Int64) \ - M(Float32) \ - M(Float64) - -#define FOR_NUMERIC_TYPES(M) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(UInt128) \ - M(UInt256) \ - M(Int8) \ - M(Int16) \ - M(Int32) \ - M(Int64) \ - M(Int128) \ - M(Int256) \ - M(Float32) \ - M(Float64) - namespace DB { struct Settings; diff --git a/src/Backups/BackupCoordinationDistributed.cpp b/src/Backups/BackupCoordinationDistributed.cpp index 945239482fc..5b932229e71 100644 --- a/src/Backups/BackupCoordinationDistributed.cpp +++ b/src/Backups/BackupCoordinationDistributed.cpp @@ -131,7 +131,7 @@ namespace BackupCoordinationDistributed::BackupCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , stage_sync(zookeeper_path_ + "/stage", get_zookeeper_, &Poco::Logger::get("BackupCoordination")) + , status_sync(zookeeper_path_ + "/status", get_zookeeper_, &Poco::Logger::get("BackupCoordination")) { createRootNodes(); } @@ -145,6 +145,8 @@ void BackupCoordinationDistributed::createRootNodes() zookeeper->createIfNotExists(zookeeper_path, ""); zookeeper->createIfNotExists(zookeeper_path + "/repl_part_names", ""); zookeeper->createIfNotExists(zookeeper_path + "/repl_data_paths", ""); + zookeeper->createIfNotExists(zookeeper_path + "/repl_access_host", ""); + zookeeper->createIfNotExists(zookeeper_path + "/repl_access_paths", ""); zookeeper->createIfNotExists(zookeeper_path + "/file_names", ""); zookeeper->createIfNotExists(zookeeper_path + "/file_infos", ""); zookeeper->createIfNotExists(zookeeper_path + "/archive_suffixes", ""); @@ -157,19 +159,24 @@ void BackupCoordinationDistributed::removeAllNodes() } -void BackupCoordinationDistributed::syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) +void BackupCoordinationDistributed::setStatus(const String & current_host, const String & new_status, const String & message) { - stage_sync.syncStage(current_host, new_stage, wait_hosts, timeout); + status_sync.set(current_host, new_status, message); } -void BackupCoordinationDistributed::syncStageError(const String & current_host, const String & error_message) +Strings BackupCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) { - stage_sync.syncStageError(current_host, error_message); + return status_sync.setAndWait(current_host, new_status, message, all_hosts); +} + +Strings BackupCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) +{ + return status_sync.setAndWaitFor(current_host, new_status, message, all_hosts, timeout_ms); } void BackupCoordinationDistributed::addReplicatedPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) @@ -181,39 +188,39 @@ void BackupCoordinationDistributed::addReplicatedPartNames( } auto zookeeper = get_zookeeper(); - String path = zookeeper_path + "/repl_part_names/" + escapeForFileName(table_zk_path); + String path = zookeeper_path + "/repl_part_names/" + escapeForFileName(table_shared_id); zookeeper->createIfNotExists(path, ""); path += "/" + escapeForFileName(replica_name); zookeeper->create(path, ReplicatedPartNames::serialize(part_names_and_checksums, table_name_for_logs), zkutil::CreateMode::Persistent); } -Strings BackupCoordinationDistributed::getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const +Strings BackupCoordinationDistributed::getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const { std::lock_guard lock{mutex}; prepareReplicatedPartNames(); - return replicated_part_names->getPartNames(table_zk_path, replica_name); + return replicated_part_names->getPartNames(table_shared_id, replica_name); } void BackupCoordinationDistributed::addReplicatedDataPath( - const String & table_zk_path, const String & data_path) + const String & table_shared_id, const String & data_path) { auto zookeeper = get_zookeeper(); - String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_zk_path); + String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_shared_id); + zookeeper->createIfNotExists(path, ""); + path += "/" + escapeForFileName(data_path); zookeeper->createIfNotExists(path, ""); - path += "/"; - zookeeper->create(path, data_path, zkutil::CreateMode::PersistentSequential); } -Strings BackupCoordinationDistributed::getReplicatedDataPaths(const String & table_zk_path) const +Strings BackupCoordinationDistributed::getReplicatedDataPaths(const String & table_shared_id) const { auto zookeeper = get_zookeeper(); - String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_zk_path); + String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_shared_id); Strings children = zookeeper->getChildren(path); Strings data_paths; data_paths.reserve(children.size()); for (const String & child : children) - data_paths.push_back(zookeeper->get(path + "/" + child)); + data_paths.push_back(unescapeForFileName(child)); return data_paths; } @@ -240,6 +247,47 @@ void BackupCoordinationDistributed::prepareReplicatedPartNames() const } +void BackupCoordinationDistributed::addReplicatedAccessPath(const String & access_zk_path, const String & file_path) +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_paths/" + escapeForFileName(access_zk_path); + zookeeper->createIfNotExists(path, ""); + path += "/" + escapeForFileName(file_path); + zookeeper->createIfNotExists(path, ""); +} + +Strings BackupCoordinationDistributed::getReplicatedAccessPaths(const String & access_zk_path) const +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_paths/" + escapeForFileName(access_zk_path); + Strings children = zookeeper->getChildren(path); + Strings file_paths; + file_paths.reserve(children.size()); + for (const String & child : children) + file_paths.push_back(unescapeForFileName(child)); + return file_paths; +} + +void BackupCoordinationDistributed::setReplicatedAccessHost(const String & access_zk_path, const String & host_id) +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_host/" + escapeForFileName(access_zk_path); + auto code = zookeeper->tryCreate(path, host_id, zkutil::CreateMode::Persistent); + if ((code != Coordination::Error::ZOK) && (code != Coordination::Error::ZNODEEXISTS)) + throw zkutil::KeeperException(code, path); + + if (code == Coordination::Error::ZNODEEXISTS) + zookeeper->set(path, host_id); +} + +String BackupCoordinationDistributed::getReplicatedAccessHost(const String & access_zk_path) const +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_host/" + escapeForFileName(access_zk_path); + return zookeeper->get(path); +} + + void BackupCoordinationDistributed::addFileInfo(const FileInfo & file_info, bool & is_data_file_required) { auto zookeeper = get_zookeeper(); diff --git a/src/Backups/BackupCoordinationDistributed.h b/src/Backups/BackupCoordinationDistributed.h index 2872e1f3ae4..813132bd0b8 100644 --- a/src/Backups/BackupCoordinationDistributed.h +++ b/src/Backups/BackupCoordinationDistributed.h @@ -14,19 +14,26 @@ public: BackupCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_); ~BackupCoordinationDistributed() override; - void syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - void syncStageError(const String & current_host, const String & error_message) override; + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; void addReplicatedPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; - Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const override; + Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const override; - void addReplicatedDataPath(const String & table_zk_path, const String & data_path) override; - Strings getReplicatedDataPaths(const String & table_zk_path) const override; + void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override; + Strings getReplicatedDataPaths(const String & table_shared_id) const override; + + void addReplicatedAccessPath(const String & access_zk_path, const String & file_path) override; + Strings getReplicatedAccessPaths(const String & access_zk_path) const override; + + void setReplicatedAccessHost(const String & access_zk_path, const String & host_id) override; + String getReplicatedAccessHost(const String & access_zk_path) const override; void addFileInfo(const FileInfo & file_info, bool & is_data_file_required) override; void updateFileInfo(const FileInfo & file_info) override; @@ -51,7 +58,7 @@ private: const String zookeeper_path; const zkutil::GetZooKeeper get_zookeeper; - BackupCoordinationStageSync stage_sync; + BackupCoordinationStatusSync status_sync; mutable std::mutex mutex; mutable std::optional replicated_part_names; diff --git a/src/Backups/BackupCoordinationHelpers.cpp b/src/Backups/BackupCoordinationHelpers.cpp index 9528f888770..7f570ba9c85 100644 --- a/src/Backups/BackupCoordinationHelpers.cpp +++ b/src/Backups/BackupCoordinationHelpers.cpp @@ -157,7 +157,7 @@ BackupCoordinationReplicatedPartNames::BackupCoordinationReplicatedPartNames() = BackupCoordinationReplicatedPartNames::~BackupCoordinationReplicatedPartNames() = default; void BackupCoordinationReplicatedPartNames::addPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) @@ -165,7 +165,7 @@ void BackupCoordinationReplicatedPartNames::addPartNames( if (part_names_prepared) throw Exception(ErrorCodes::LOGICAL_ERROR, "addPartNames() must not be called after getPartNames()"); - auto & table_info = table_infos[table_zk_path]; + auto & table_info = table_infos[table_shared_id]; if (!table_info.covered_parts_finder) table_info.covered_parts_finder = std::make_unique(table_name_for_logs); @@ -207,10 +207,10 @@ void BackupCoordinationReplicatedPartNames::addPartNames( } } -Strings BackupCoordinationReplicatedPartNames::getPartNames(const String & table_zk_path, const String & replica_name) const +Strings BackupCoordinationReplicatedPartNames::getPartNames(const String & table_shared_id, const String & replica_name) const { preparePartNames(); - auto it = table_infos.find(table_zk_path); + auto it = table_infos.find(table_shared_id); if (it == table_infos.end()) return {}; const auto & replicas_parts = it->second.replicas_parts; @@ -243,7 +243,7 @@ void BackupCoordinationReplicatedPartNames::preparePartNames() const /// Helps to wait until all hosts come to a specified stage. -BackupCoordinationStageSync::BackupCoordinationStageSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_) +BackupCoordinationStatusSync::BackupCoordinationStatusSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) , log(log_) @@ -251,63 +251,78 @@ BackupCoordinationStageSync::BackupCoordinationStageSync(const String & zookeepe createRootNodes(); } -void BackupCoordinationStageSync::createRootNodes() +void BackupCoordinationStatusSync::createRootNodes() { auto zookeeper = get_zookeeper(); zookeeper->createAncestors(zookeeper_path); zookeeper->createIfNotExists(zookeeper_path, ""); } -void BackupCoordinationStageSync::syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) +void BackupCoordinationStatusSync::set(const String & current_host, const String & new_status, const String & message) { - /// Put new stage to ZooKeeper. - auto zookeeper = get_zookeeper(); - zookeeper->createIfNotExists(zookeeper_path + "/" + current_host + "|" + std::to_string(new_stage), ""); + setImpl(current_host, new_status, message, {}, {}); +} - if (wait_hosts.empty() || ((wait_hosts.size() == 1) && (wait_hosts.front() == current_host))) - return; +Strings BackupCoordinationStatusSync::setAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) +{ + return setImpl(current_host, new_status, message, all_hosts, {}); +} + +Strings BackupCoordinationStatusSync::setAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) +{ + return setImpl(current_host, new_status, message, all_hosts, timeout_ms); +} + +Strings BackupCoordinationStatusSync::setImpl(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, const std::optional & timeout_ms) +{ + /// Put new status to ZooKeeper. + auto zookeeper = get_zookeeper(); + zookeeper->createIfNotExists(zookeeper_path + "/" + current_host + "|" + new_status, message); + + if (all_hosts.empty() || (new_status == kErrorStatus)) + return {}; + + if ((all_hosts.size() == 1) && (all_hosts.front() == current_host)) + return {message}; /// Wait for other hosts. - /// Current stages of all hosts. + Strings ready_hosts_results; + ready_hosts_results.resize(all_hosts.size()); + + std::map /* index in `ready_hosts_results` */> unready_hosts; + for (size_t i = 0; i != all_hosts.size(); ++i) + unready_hosts[all_hosts[i]].push_back(i); + std::optional host_with_error; std::optional error_message; - std::map> unready_hosts; - for (const String & host : wait_hosts) - unready_hosts.emplace(host, std::optional{}); - /// Process ZooKeeper's nodes and set `all_hosts_ready` or `unready_host` or `error_message`. auto process_zk_nodes = [&](const Strings & zk_nodes) { for (const String & zk_node : zk_nodes) { - if (zk_node == "error") + if (zk_node.starts_with("remove_watch-")) + continue; + + size_t separator_pos = zk_node.find('|'); + if (separator_pos == String::npos) + throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected zk node {}", zookeeper_path + "/" + zk_node); + String host = zk_node.substr(0, separator_pos); + String status = zk_node.substr(separator_pos + 1); + if (status == kErrorStatus) { - String str = zookeeper->get(zookeeper_path + "/" + zk_node); - size_t separator_pos = str.find('|'); - if (separator_pos == String::npos) - throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected value of zk node {}: {}", zookeeper_path + "/" + zk_node, str); - host_with_error = str.substr(0, separator_pos); - error_message = str.substr(separator_pos + 1); + host_with_error = host; + error_message = zookeeper->get(zookeeper_path + "/" + zk_node); return; } - else if (!zk_node.starts_with("remove_watch-")) + auto it = unready_hosts.find(host); + if ((it != unready_hosts.end()) && (status == new_status)) { - size_t separator_pos = zk_node.find('|'); - if (separator_pos == String::npos) - throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected zk node {}", zookeeper_path + "/" + zk_node); - String host = zk_node.substr(0, separator_pos); - int found_stage = parseFromString(zk_node.substr(separator_pos + 1)); - auto it = unready_hosts.find(host); - if (it != unready_hosts.end()) - { - auto & stage = it->second; - if (!stage || (stage < found_stage)) - stage = found_stage; - if (stage >= new_stage) - unready_hosts.erase(it); - } + String result = zookeeper->get(zookeeper_path + "/" + zk_node); + for (size_t i : it->second) + ready_hosts_results[i] = result; + unready_hosts.erase(it); } } }; @@ -324,7 +339,8 @@ void BackupCoordinationStageSync::syncStage(const String & current_host, int new auto watch_triggered = [&] { return !watch_set; }; - bool use_timeout = (timeout.count() >= 0); + bool use_timeout = timeout_ms.has_value(); + std::chrono::milliseconds timeout{timeout_ms.value_or(0)}; std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now(); std::chrono::steady_clock::duration elapsed; std::mutex dummy_mutex; @@ -369,12 +385,8 @@ void BackupCoordinationStageSync::syncStage(const String & current_host, int new unready_hosts.begin()->first, to_string(elapsed)); } -} -void BackupCoordinationStageSync::syncStageError(const String & current_host, const String & error_message) -{ - auto zookeeper = get_zookeeper(); - zookeeper->createIfNotExists(zookeeper_path + "/error", current_host + "|" + error_message); + return ready_hosts_results; } } diff --git a/src/Backups/BackupCoordinationHelpers.h b/src/Backups/BackupCoordinationHelpers.h index b0cd0440b98..2e9e4b3cbde 100644 --- a/src/Backups/BackupCoordinationHelpers.h +++ b/src/Backups/BackupCoordinationHelpers.h @@ -24,7 +24,7 @@ public: /// getPartNames(). /// Checksums are used only to control that parts under the same names on different replicas are the same. void addPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums); @@ -32,7 +32,7 @@ public: /// Returns the names of the parts which a specified replica of a replicated table should put to the backup. /// This is the same list as it was added by call of the function addPartNames() but without duplications and without /// parts covered by another parts. - Strings getPartNames(const String & table_zk_path, const String & replica_name) const; + Strings getPartNames(const String & table_shared_id, const String & replica_name) const; private: void preparePartNames() const; @@ -52,22 +52,26 @@ private: std::unique_ptr covered_parts_finder; }; - std::map table_infos; /// Should be ordered because we need this map to be in the same order on every replica. + std::map table_infos; /// Should be ordered because we need this map to be in the same order on every replica. mutable bool part_names_prepared = false; }; /// Helps to wait until all hosts come to a specified stage. -class BackupCoordinationStageSync +class BackupCoordinationStatusSync { public: - BackupCoordinationStageSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_); + BackupCoordinationStatusSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_); - void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout); - void syncStageError(const String & current_host, const String & error_message); + void set(const String & current_host, const String & new_status, const String & message); + Strings setAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts); + Strings setAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms); + + static constexpr const char * kErrorStatus = "error"; private: void createRootNodes(); + Strings setImpl(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, const std::optional & timeout_ms); String zookeeper_path; zkutil::GetZooKeeper get_zookeeper; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 55a3c671a6e..a7d5602ca30 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -13,43 +13,80 @@ using FileInfo = IBackupCoordination::FileInfo; BackupCoordinationLocal::BackupCoordinationLocal() = default; BackupCoordinationLocal::~BackupCoordinationLocal() = default; -void BackupCoordinationLocal::syncStage(const String &, int, const Strings &, std::chrono::seconds) +void BackupCoordinationLocal::setStatus(const String &, const String &, const String &) { } -void BackupCoordinationLocal::syncStageError(const String &, const String &) +Strings BackupCoordinationLocal::setStatusAndWait(const String &, const String &, const String &, const Strings &) { + return {}; } -void BackupCoordinationLocal::addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) +Strings BackupCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const String &, const Strings &, UInt64) +{ + return {}; +} + +void BackupCoordinationLocal::addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) { std::lock_guard lock{mutex}; - replicated_part_names.addPartNames(table_zk_path, table_name_for_logs, replica_name, part_names_and_checksums); + replicated_part_names.addPartNames(table_shared_id, table_name_for_logs, replica_name, part_names_and_checksums); } -Strings BackupCoordinationLocal::getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const +Strings BackupCoordinationLocal::getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const { std::lock_guard lock{mutex}; - return replicated_part_names.getPartNames(table_zk_path, replica_name); + return replicated_part_names.getPartNames(table_shared_id, replica_name); } -void BackupCoordinationLocal::addReplicatedDataPath(const String & table_zk_path, const String & data_path) +void BackupCoordinationLocal::addReplicatedDataPath(const String & table_shared_id, const String & data_path) { std::lock_guard lock{mutex}; - replicated_data_paths[table_zk_path].push_back(data_path); + replicated_data_paths[table_shared_id].push_back(data_path); } -Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_zk_path) const +Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_shared_id) const { std::lock_guard lock{mutex}; - auto it = replicated_data_paths.find(table_zk_path); + auto it = replicated_data_paths.find(table_shared_id); if (it == replicated_data_paths.end()) return {}; return it->second; } +void BackupCoordinationLocal::addReplicatedAccessPath(const String & access_zk_path, const String & file_path) +{ + std::lock_guard lock{mutex}; + replicated_access_paths[access_zk_path].push_back(file_path); +} + +Strings BackupCoordinationLocal::getReplicatedAccessPaths(const String & access_zk_path) const +{ + std::lock_guard lock{mutex}; + auto it = replicated_access_paths.find(access_zk_path); + if (it == replicated_access_paths.end()) + return {}; + return it->second; +} + +void BackupCoordinationLocal::setReplicatedAccessHost(const String & access_zk_path, const String & host_id) +{ + std::lock_guard lock{mutex}; + replicated_access_hosts[access_zk_path] = host_id; +} + +String BackupCoordinationLocal::getReplicatedAccessHost(const String & access_zk_path) const +{ + std::lock_guard lock{mutex}; + auto it = replicated_access_hosts.find(access_zk_path); + if (it == replicated_access_hosts.end()) + return {}; + return it->second; +} + + void BackupCoordinationLocal::addFileInfo(const FileInfo & file_info, bool & is_data_file_required) { std::lock_guard lock{mutex}; diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 6529184c61a..dcd6505a438 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -19,15 +19,22 @@ public: BackupCoordinationLocal(); ~BackupCoordinationLocal() override; - void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - void syncStageError(const String & current_host, const String & error_message) override; + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; - void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, + void addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; - Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const override; + Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const override; - void addReplicatedDataPath(const String & table_zk_path, const String & data_path) override; - Strings getReplicatedDataPaths(const String & table_zk_path) const override; + void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override; + Strings getReplicatedDataPaths(const String & table_shared_id) const override; + + void addReplicatedAccessPath(const String & access_zk_path, const String & file_path) override; + Strings getReplicatedAccessPaths(const String & access_zk_path) const override; + + void setReplicatedAccessHost(const String & access_zk_path, const String & host_id) override; + String getReplicatedAccessHost(const String & access_zk_path) const override; void addFileInfo(const FileInfo & file_info, bool & is_data_file_required) override; void updateFileInfo(const FileInfo & file_info) override; @@ -47,6 +54,8 @@ private: mutable std::mutex mutex; BackupCoordinationReplicatedPartNames replicated_part_names TSA_GUARDED_BY(mutex); std::unordered_map replicated_data_paths TSA_GUARDED_BY(mutex); + std::unordered_map replicated_access_paths TSA_GUARDED_BY(mutex); + std::unordered_map replicated_access_hosts TSA_GUARDED_BY(mutex); std::map file_names TSA_GUARDED_BY(mutex); /// Should be ordered alphabetically, see listFiles(). For empty files we assume checksum = 0. std::map file_infos TSA_GUARDED_BY(mutex); /// Information about files. Without empty files. Strings archive_suffixes TSA_GUARDED_BY(mutex); diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 322bc00ee3c..d5ed9e0da2b 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -1,14 +1,18 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include +#include #include #include #include @@ -21,35 +25,63 @@ namespace DB namespace ErrorCodes { - extern const int CANNOT_COLLECT_OBJECTS_FOR_BACKUP; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; extern const int CANNOT_BACKUP_TABLE; extern const int TABLE_IS_DROPPED; + extern const int UNKNOWN_TABLE; extern const int LOGICAL_ERROR; } - -bool BackupEntriesCollector::TableKey::operator ==(const TableKey & right) const +namespace { - return (name == right.name) && (is_temporary == right.is_temporary); -} + /// Finding all tables and databases which we're going to put to the backup and collecting their metadata. + constexpr const char * kGatheringMetadataStatus = "gathering metadata"; -bool BackupEntriesCollector::TableKey::operator <(const TableKey & right) const -{ - return (name < right.name) || ((name == right.name) && (is_temporary < right.is_temporary)); -} + /// Making temporary hard links and prepare backup entries. + constexpr const char * kExtractingDataFromTablesStatus = "extracting data from tables"; -std::string_view BackupEntriesCollector::toString(Stage stage) -{ - switch (stage) + /// Running special tasks for replicated tables which can also prepare some backup entries. + constexpr const char * kRunningPostTasksStatus = "running post-tasks"; + + /// Writing backup entries to the backup and removing temporary hard links. + constexpr const char * kWritingBackupStatus = "writing backup"; + + /// Error status. + constexpr const char * kErrorStatus = BackupCoordinationStatusSync::kErrorStatus; + + /// Uppercases the first character of a passed string. + String toUpperFirst(const String & str) { - case Stage::kPreparing: return "Preparing"; - case Stage::kFindingTables: return "Finding tables"; - case Stage::kExtractingDataFromTables: return "Extracting data from tables"; - case Stage::kRunningPostTasks: return "Running post tasks"; - case Stage::kWritingBackup: return "Writing backup"; - case Stage::kError: return "Error"; + String res = str; + res[0] = std::toupper(res[0]); + return res; + } + + /// Outputs "table " or "temporary table " + String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_upper) + { + String str; + if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) + str = fmt::format("temporary table {}", backQuoteIfNeed(table_name)); + else + str = fmt::format("table {}.{}", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + if (first_upper) + str[0] = std::toupper(str[0]); + return str; + } + + /// How long we should sleep after finding an inconsistency error. + std::chrono::milliseconds getSleepTimeAfterInconsistencyError(size_t pass) + { + size_t ms; + if (pass == 1) /* pass is 1-based */ + ms = 0; + else if ((pass % 10) != 1) + ms = 0; + else + ms = 1000; + return std::chrono::milliseconds{ms}; } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown backup stage: {}", static_cast(stage)); } @@ -57,36 +89,38 @@ BackupEntriesCollector::BackupEntriesCollector( const ASTBackupQuery::Elements & backup_query_elements_, const BackupSettings & backup_settings_, std::shared_ptr backup_coordination_, - const ContextPtr & context_, - std::chrono::seconds timeout_) + const ContextPtr & context_) : backup_query_elements(backup_query_elements_) , backup_settings(backup_settings_) , backup_coordination(backup_coordination_) , context(context_) - , timeout(timeout_) + , consistent_metadata_snapshot_timeout(context->getConfigRef().getUInt64("backups.consistent_metadata_snapshot_timeout", 300000)) , log(&Poco::Logger::get("BackupEntriesCollector")) { } BackupEntriesCollector::~BackupEntriesCollector() = default; -BackupEntries BackupEntriesCollector::getBackupEntries() +BackupEntries BackupEntriesCollector::run() { try { - /// getBackupEntries() must not be called multiple times. - if (current_stage != Stage::kPreparing) + /// run() can be called onle once. + if (!current_status.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Already making backup entries"); - /// Calculate the root path for collecting backup entries, it's either empty or has the format "shards//replicas//". - calculateRootPathInBackup(); + /// Find other hosts working along with us to execute this ON CLUSTER query. + all_hosts + = BackupSettings::Util::filterHostIDs(backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); /// Do renaming in the create queries according to the renaming config. renaming_map = makeRenamingMapFromBackupQuery(backup_query_elements); + /// Calculate the root path for collecting backup entries, it's either empty or has the format "shards//replicas//". + calculateRootPathInBackup(); + /// Find databases and tables which we're going to put to the backup. - setStage(Stage::kFindingTables); - collectDatabasesAndTablesInfo(); + gatherMetadataAndCheckConsistency(); /// Make backup entries for the definitions of the found databases. makeBackupEntriesForDatabasesDefs(); @@ -95,15 +129,15 @@ BackupEntries BackupEntriesCollector::getBackupEntries() makeBackupEntriesForTablesDefs(); /// Make backup entries for the data of the found tables. - setStage(Stage::kExtractingDataFromTables); + setStatus(kExtractingDataFromTablesStatus); makeBackupEntriesForTablesData(); /// Run all the tasks added with addPostCollectingTask(). - setStage(Stage::kRunningPostTasks); - runPostCollectingTasks(); + setStatus(kRunningPostTasksStatus); + runPostTasks(); /// No more backup entries or tasks are allowed after this point. - setStage(Stage::kWritingBackup); + setStatus(kWritingBackupStatus); return std::move(backup_entries); } @@ -111,7 +145,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() { try { - setStage(Stage::kError, getCurrentExceptionMessage(false)); + setStatus(kErrorStatus, getCurrentExceptionMessage(false)); } catch (...) { @@ -120,24 +154,34 @@ BackupEntries BackupEntriesCollector::getBackupEntries() } } -void BackupEntriesCollector::setStage(Stage new_stage, const String & error_message) +Strings BackupEntriesCollector::setStatus(const String & new_status, const String & message) { - if (new_stage == Stage::kError) - LOG_ERROR(log, "{} failed with error: {}", toString(current_stage), error_message); - else - LOG_TRACE(log, "{}", toString(new_stage)); - - current_stage = new_stage; - - if (new_stage == Stage::kError) + if (new_status == kErrorStatus) { - backup_coordination->syncStageError(backup_settings.host_id, error_message); + LOG_ERROR(log, "{} failed with error: {}", toUpperFirst(current_status), message); + backup_coordination->setStatus(backup_settings.host_id, new_status, message); + return {}; } else { - auto all_hosts - = BackupSettings::Util::filterHostIDs(backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); - backup_coordination->syncStage(backup_settings.host_id, static_cast(new_stage), all_hosts, timeout); + LOG_TRACE(log, "{}", toUpperFirst(new_status)); + current_status = new_status; + if (new_status.starts_with(kGatheringMetadataStatus)) + { + auto now = std::chrono::steady_clock::now(); + auto end_of_timeout = std::max(now, consistent_metadata_snapshot_start_time + consistent_metadata_snapshot_timeout); + + return backup_coordination->setStatusAndWaitFor( + backup_settings.host_id, + new_status, + message, + all_hosts, + std::chrono::duration_cast(end_of_timeout - now).count()); + } + else + { + return backup_coordination->setStatusAndWait(backup_settings.host_id, new_status, message, all_hosts); + } } } @@ -156,287 +200,444 @@ void BackupEntriesCollector::calculateRootPathInBackup() } /// Finds databases and tables which we will put to the backup. -void BackupEntriesCollector::collectDatabasesAndTablesInfo() +void BackupEntriesCollector::gatherMetadataAndCheckConsistency() { - bool use_timeout = (timeout.count() >= 0); - auto start_time = std::chrono::steady_clock::now(); + consistent_metadata_snapshot_start_time = std::chrono::steady_clock::now(); + auto end_of_timeout = consistent_metadata_snapshot_start_time + consistent_metadata_snapshot_timeout; + setStatus(fmt::format("{} ({})", kGatheringMetadataStatus, 1)); - int pass = 0; - do + for (size_t pass = 1;; ++pass) { - database_infos.clear(); - table_infos.clear(); - consistent = true; - - /// Collect information about databases and tables specified in the BACKUP query. - for (const auto & element : backup_query_elements) + String new_status = fmt::format("{} ({})", kGatheringMetadataStatus, pass + 1); + std::optional inconsistency_error; + if (tryGatherMetadataAndCompareWithPrevious(inconsistency_error)) { - switch (element.type) + /// Gathered metadata and checked consistency, cool! But we have to check that other hosts cope with that too. + auto all_hosts_results = setStatus(new_status, "consistent"); + + std::optional host_with_inconsistency; + std::optional inconsistency_error_on_other_host; + for (size_t i = 0; i != all_hosts.size(); ++i) { - case ASTBackupQuery::ElementType::TABLE: + if ((i < all_hosts_results.size()) && (all_hosts_results[i] != "consistent")) { - collectTableInfo({element.database_name, element.table_name}, false, element.partitions, true); - break; - } - - case ASTBackupQuery::ElementType::TEMPORARY_TABLE: - { - collectTableInfo({"", element.table_name}, true, element.partitions, true); - break; - } - - case ASTBackupQuery::ElementType::DATABASE: - { - collectDatabaseInfo(element.database_name, element.except_tables, true); - break; - } - - case ASTBackupQuery::ElementType::ALL: - { - collectAllDatabasesInfo(element.except_databases, element.except_tables); + host_with_inconsistency = all_hosts[i]; + inconsistency_error_on_other_host = all_hosts_results[i]; break; } } + + if (!host_with_inconsistency) + break; /// All hosts managed to gather metadata and everything is consistent, so we can go further to writing the backup. + + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "Found inconsistency on host {}: {}", + *host_with_inconsistency, + *inconsistency_error_on_other_host}; } - - /// We have to check consistency of collected information to protect from the case when some table or database is - /// renamed during this collecting making the collected information invalid. - checkConsistency(); - - /// Two passes is absolute minimum (see `previous_table_names` & `previous_database_names`). - auto elapsed = std::chrono::steady_clock::now() - start_time; - if (!consistent && (pass >= 2) && use_timeout) + else { - if (elapsed > timeout) - throw Exception( - ErrorCodes::CANNOT_COLLECT_OBJECTS_FOR_BACKUP, - "Couldn't collect tables and databases to make a backup (pass #{}, elapsed {})", - pass, - to_string(elapsed)); + /// Failed to gather metadata or something wasn't consistent. We'll let other hosts know that and try again. + setStatus(new_status, inconsistency_error->displayText()); } + /// Two passes is minimum (we need to compare with table names with previous ones to be sure we don't miss anything). if (pass >= 2) - LOG_WARNING(log, "Couldn't collect tables and databases to make a backup (pass #{}, elapsed {})", pass, to_string(elapsed)); - ++pass; - } while (!consistent); + { + if (std::chrono::steady_clock::now() > end_of_timeout) + inconsistency_error->rethrow(); + else + LOG_WARNING(log, "{}", inconsistency_error->displayText()); + } + + auto sleep_time = getSleepTimeAfterInconsistencyError(pass); + if (sleep_time.count() > 0) + sleepForNanoseconds(std::chrono::duration_cast(sleep_time).count()); + } LOG_INFO(log, "Will backup {} databases and {} tables", database_infos.size(), table_infos.size()); } -void BackupEntriesCollector::collectTableInfo( - const QualifiedTableName & table_name, bool is_temporary_table, const std::optional & partitions, bool throw_if_not_found) +bool BackupEntriesCollector::tryGatherMetadataAndCompareWithPrevious(std::optional & inconsistency_error) { - /// Gather information about the table. - DatabasePtr database; - StoragePtr storage; - TableLockHolder table_lock; - ASTPtr create_table_query; - - TableKey table_key{table_name, is_temporary_table}; - - if (throw_if_not_found) + try { - auto resolved_id = is_temporary_table - ? context->resolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal) - : context->resolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal); - std::tie(database, storage) = DatabaseCatalog::instance().getDatabaseAndTable(resolved_id, context); - table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - create_table_query = storage->getCreateQueryForBackup(*this); + /// Collect information about databases and tables specified in the BACKUP query. + database_infos.clear(); + table_infos.clear(); + gatherDatabasesMetadata(); + gatherTablesMetadata(); } - else + catch (Exception & e) { - auto resolved_id = is_temporary_table - ? context->tryResolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal) - : context->tryResolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal); - if (!resolved_id.empty()) - std::tie(database, storage) = DatabaseCatalog::instance().tryGetDatabaseAndTable(resolved_id, context); + if (e.code() != ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP) + throw; + inconsistency_error = e; + return false; + } + + /// We have to check consistency of collected information to protect from the case when some table or database is + /// renamed during this collecting making the collected information invalid. + return compareWithPrevious(inconsistency_error); +} + +void BackupEntriesCollector::gatherDatabasesMetadata() +{ + /// Collect information about databases and tables specified in the BACKUP query. + for (const auto & element : backup_query_elements) + { + switch (element.type) + { + case ASTBackupQuery::ElementType::TABLE: + { + gatherDatabaseMetadata( + element.database_name, + /* throw_if_database_not_found= */ true, + /* backup_create_database_query= */ false, + element.table_name, + /* throw_if_table_not_found= */ true, + element.partitions, + /* all_tables= */ false, + /* except_table_names= */ {}); + break; + } + + case ASTBackupQuery::ElementType::TEMPORARY_TABLE: + { + gatherDatabaseMetadata( + DatabaseCatalog::TEMPORARY_DATABASE, + /* throw_if_database_not_found= */ true, + /* backup_create_database_query= */ false, + element.table_name, + /* throw_if_table_not_found= */ true, + element.partitions, + /* all_tables= */ false, + /* except_table_names= */ {}); + break; + } + + case ASTBackupQuery::ElementType::DATABASE: + { + gatherDatabaseMetadata( + element.database_name, + /* throw_if_database_not_found= */ true, + /* backup_create_database_query= */ true, + /* table_name= */ {}, + /* throw_if_table_not_found= */ false, + /* partitions= */ {}, + /* all_tables= */ true, + /* except_table_names= */ element.except_tables); + break; + } + + case ASTBackupQuery::ElementType::ALL: + { + for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases()) + { + if (!element.except_databases.contains(database_name)) + { + gatherDatabaseMetadata( + database_name, + /* throw_if_database_not_found= */ false, + /* backup_create_database_query= */ true, + /* table_name= */ {}, + /* throw_if_table_not_found= */ false, + /* partitions= */ {}, + /* all_tables= */ true, + /* except_table_names= */ element.except_tables); + } + } + break; + } + } + } +} + +void BackupEntriesCollector::gatherDatabaseMetadata( + const String & database_name, + bool throw_if_database_not_found, + bool backup_create_database_query, + const std::optional & table_name, + bool throw_if_table_not_found, + const std::optional & partitions, + bool all_tables, + const std::set & except_table_names) +{ + auto it = database_infos.find(database_name); + if (it == database_infos.end()) + { + DatabasePtr database; + if (throw_if_database_not_found) + { + database = DatabaseCatalog::instance().getDatabase(database_name); + } + else + { + database = DatabaseCatalog::instance().tryGetDatabase(database_name); + if (!database) + return; + } + + DatabaseInfo new_database_info; + new_database_info.database = database; + it = database_infos.emplace(database_name, new_database_info).first; + } + + DatabaseInfo & database_info = it->second; + + if (backup_create_database_query && !database_info.create_database_query && (database_name != DatabaseCatalog::TEMPORARY_DATABASE)) + { + ASTPtr create_database_query; + try + { + create_database_query = database_info.database->getCreateDatabaseQuery(); + } + catch (...) + { + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for database {}", database_name); + } + + database_info.create_database_query = create_database_query; + const auto & create = create_database_query->as(); + + if (create.getDatabase() != database_name) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for database {}", backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(database_name)); + + String new_database_name = renaming_map.getNewDatabaseName(database_name); + database_info.metadata_path_in_backup = root_path_in_backup / "metadata" / (escapeForFileName(new_database_name) + ".sql"); + } + + if (table_name) + { + auto & table_params = database_info.tables[*table_name]; + if (throw_if_table_not_found) + table_params.throw_if_table_not_found = true; + if (partitions) + { + table_params.partitions.emplace(); + insertAtEnd(*table_params.partitions, *partitions); + } + database_info.except_table_names.emplace(*table_name); + } + + if (all_tables) + { + database_info.all_tables = all_tables; + for (const auto & except_table_name : except_table_names) + if (except_table_name.first == database_name) + database_info.except_table_names.emplace(except_table_name.second); + } +} + +void BackupEntriesCollector::gatherTablesMetadata() +{ + table_infos.clear(); + for (const auto & [database_name, database_info] : database_infos) + { + const auto & database = database_info.database; + bool is_temporary_database = (database_name == DatabaseCatalog::TEMPORARY_DATABASE); + + auto filter_by_table_name = [database_info = &database_info](const String & table_name) + { + /// We skip inner tables of materialized views. + if (table_name.starts_with(".inner_id.")) + return false; + + if (database_info->tables.contains(table_name)) + return true; + + if (database_info->all_tables) + return !database_info->except_table_names.contains(table_name); + + return false; + }; + + auto db_tables = database->getTablesForBackup(filter_by_table_name, context); + + std::unordered_set found_table_names; + for (const auto & db_table : db_tables) + { + const auto & create_table_query = db_table.first; + const auto & create = create_table_query->as(); + found_table_names.emplace(create.getTable()); + + if (is_temporary_database && !create.temporary) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a non-temporary create query for {}", tableNameWithTypeToString(database_name, create.getTable(), false)); + + if (!is_temporary_database && (create.getDatabase() != database_name)) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected database name {} for {}", backQuoteIfNeed(create.getDatabase()), tableNameWithTypeToString(database_name, create.getTable(), false)); + } + + /// Check that all tables were found. + for (const auto & [table_name, table_info] : database_info.tables) + { + if (table_info.throw_if_table_not_found && !found_table_names.contains(table_name)) + throw Exception(ErrorCodes::UNKNOWN_TABLE, "{} not found", tableNameWithTypeToString(database_name, table_name, true)); + } + + for (const auto & db_table : db_tables) + { + const auto & create_table_query = db_table.first; + const auto & storage = db_table.second; + const auto & create = create_table_query->as(); + String table_name = create.getTable(); + + fs::path metadata_path_in_backup, data_path_in_backup; + auto table_name_in_backup = renaming_map.getNewTableName({database_name, table_name}); + if (table_name_in_backup.database == DatabaseCatalog::TEMPORARY_DATABASE) + { + metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(table_name_in_backup.table) + ".sql"); + data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup.table); + } + else + { + metadata_path_in_backup + = root_path_in_backup / "metadata" / escapeForFileName(table_name_in_backup.database) / (escapeForFileName(table_name_in_backup.table) + ".sql"); + data_path_in_backup = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) + / escapeForFileName(table_name_in_backup.table); + } + + /// Add information to `table_infos`. + auto & res_table_info = table_infos[QualifiedTableName{database_name, table_name}]; + res_table_info.database = database; + res_table_info.storage = storage; + res_table_info.create_table_query = create_table_query; + res_table_info.metadata_path_in_backup = metadata_path_in_backup; + res_table_info.data_path_in_backup = data_path_in_backup; + + if (!backup_settings.structure_only) + { + auto it = database_info.tables.find(table_name); + if (it != database_info.tables.end()) + { + const auto & partitions = it->second.partitions; + if (partitions && !storage->supportsBackupPartition()) + { + throw Exception( + ErrorCodes::CANNOT_BACKUP_TABLE, + "Table engine {} doesn't support partitions, cannot backup {}", + storage->getName(), + tableNameWithTypeToString(database_name, table_name, false)); + } + res_table_info.partitions = partitions; + } + } + } + } +} + +void BackupEntriesCollector::lockTablesForReading() +{ + for (auto & [table_name, table_info] : table_infos) + { + auto storage = table_info.storage; + TableLockHolder table_lock; if (storage) { try { table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - create_table_query = storage->getCreateQueryForBackup(*this); } catch (Exception & e) { if (e.code() != ErrorCodes::TABLE_IS_DROPPED) throw; - } - } - - if (!create_table_query) - { - consistent &= !table_infos.contains(table_key); - return; - } - } - - fs::path data_path_in_backup; - if (is_temporary_table) - { - auto table_name_in_backup = renaming_map.getNewTemporaryTableName(table_name.table); - data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup); - } - else - { - auto table_name_in_backup = renaming_map.getNewTableName(table_name); - data_path_in_backup - = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); - } - - /// Check that information is consistent. - const auto & create = create_table_query->as(); - if ((create.getTable() != table_name.table) || (is_temporary_table != create.temporary) || (create.getDatabase() != table_name.database)) - { - /// Table was renamed recently. - consistent = false; - return; - } - - if (auto it = table_infos.find(table_key); it != table_infos.end()) - { - const auto & table_info = it->second; - if ((table_info.database != database) || (table_info.storage != storage)) - { - /// Table was renamed recently. - consistent = false; - return; - } - } - - /// Add information to `table_infos`. - auto & res_table_info = table_infos[table_key]; - res_table_info.database = database; - res_table_info.storage = storage; - res_table_info.table_lock = table_lock; - res_table_info.create_table_query = create_table_query; - res_table_info.data_path_in_backup = data_path_in_backup; - - if (partitions) - { - if (!res_table_info.partitions) - res_table_info.partitions.emplace(); - insertAtEnd(*res_table_info.partitions, *partitions); - } -} - -void BackupEntriesCollector::collectDatabaseInfo(const String & database_name, const std::set & except_table_names, bool throw_if_not_found) -{ - /// Gather information about the database. - DatabasePtr database; - ASTPtr create_database_query; - - if (throw_if_not_found) - { - database = DatabaseCatalog::instance().getDatabase(database_name); - create_database_query = database->getCreateDatabaseQueryForBackup(); - } - else - { - database = DatabaseCatalog::instance().tryGetDatabase(database_name); - if (!database) - { - consistent &= !database_infos.contains(database_name); - return; - } - - try - { - create_database_query = database->getCreateDatabaseQueryForBackup(); - } - catch (...) - { - /// The database has been dropped recently. - consistent &= !database_infos.contains(database_name); - return; - } - } - - /// Check that information is consistent. - const auto & create = create_database_query->as(); - if (create.getDatabase() != database_name) - { - /// Database was renamed recently. - consistent = false; - return; - } - - if (auto it = database_infos.find(database_name); it != database_infos.end()) - { - const auto & database_info = it->second; - if (database_info.database != database) - { - /// Database was renamed recently. - consistent = false; - return; - } - } - - /// Add information to `database_infos`. - auto & res_database_info = database_infos[database_name]; - res_database_info.database = database; - res_database_info.create_database_query = create_database_query; - - /// Add information about tables too. - for (auto it = database->getTablesIteratorForBackup(*this); it->isValid(); it->next()) - { - if (except_table_names.contains({database_name, it->name()})) - continue; - - collectTableInfo({database_name, it->name()}, /* is_temporary_table= */ false, {}, /* throw_if_not_found= */ false); - if (!consistent) - return; - } -} - -void BackupEntriesCollector::collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names) -{ - for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases()) - { - if (except_database_names.contains(database_name)) - continue; - collectDatabaseInfo(database_name, except_table_names, false); - if (!consistent) - return; - } -} - -/// Check for consistency of collected information about databases and tables. -void BackupEntriesCollector::checkConsistency() -{ - if (!consistent) - return; /// Already inconsistent, no more checks necessary - - /// Databases found while we were scanning tables and while we were scanning databases - must be the same. - for (const auto & [key, table_info] : table_infos) - { - auto it = database_infos.find(key.name.database); - if (it != database_infos.end()) - { - const auto & database_info = it->second; - if (database_info.database != table_info.database) - { - consistent = false; - return; + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} is dropped", tableNameWithTypeToString(table_name.database, table_name.table, true)); } } } +} +/// Check consistency of collected information about databases and tables. +bool BackupEntriesCollector::compareWithPrevious(std::optional & inconsistency_error) +{ /// We need to scan tables at least twice to be sure that we haven't missed any table which could be renamed /// while we were scanning. - std::set database_names; - std::set table_names; - boost::range::copy(database_infos | boost::adaptors::map_keys, std::inserter(database_names, database_names.end())); - boost::range::copy(table_infos | boost::adaptors::map_keys, std::inserter(table_names, table_names.end())); + std::vector> databases_metadata; + std::vector> tables_metadata; + databases_metadata.reserve(database_infos.size()); + tables_metadata.reserve(table_infos.size()); + for (const auto & [database_name, database_info] : database_infos) + databases_metadata.emplace_back(database_name, database_info.create_database_query ? serializeAST(*database_info.create_database_query) : ""); + for (const auto & [table_name, table_info] : table_infos) + tables_metadata.emplace_back(table_name, serializeAST(*table_info.create_table_query)); - if (!previous_database_names || !previous_table_names || (*previous_database_names != database_names) - || (*previous_table_names != table_names)) + /// We need to sort the lists to make the comparison below correct. + ::sort(databases_metadata.begin(), databases_metadata.end()); + ::sort(tables_metadata.begin(), tables_metadata.end()); + + SCOPE_EXIT({ + previous_databases_metadata = std::move(databases_metadata); + previous_tables_metadata = std::move(tables_metadata); + }); + + /// Databases must be the same as during the previous scan. + if (databases_metadata != previous_databases_metadata) { - previous_database_names = std::move(database_names); - previous_table_names = std::move(table_names); - consistent = false; + std::vector> difference; + difference.reserve(databases_metadata.size()); + std::set_difference(databases_metadata.begin(), databases_metadata.end(), previous_databases_metadata.begin(), + previous_databases_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) + { + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "Database {} were created or changed its definition during scanning", + backQuoteIfNeed(difference[0].first)}; + return false; + } + + difference.clear(); + difference.reserve(previous_databases_metadata.size()); + std::set_difference(previous_databases_metadata.begin(), previous_databases_metadata.end(), databases_metadata.begin(), + databases_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) + { + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "Database {} were removed or changed its definition during scanning", + backQuoteIfNeed(difference[0].first)}; + return false; + } } + + /// Tables must be the same as during the previous scan. + if (tables_metadata != previous_tables_metadata) + { + std::vector> difference; + difference.reserve(tables_metadata.size()); + std::set_difference(tables_metadata.begin(), tables_metadata.end(), previous_tables_metadata.begin(), + previous_tables_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) + { + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "{} were created or changed its definition during scanning", + tableNameWithTypeToString(difference[0].first.database, difference[0].first.table, true)}; + return false; + } + + difference.clear(); + difference.reserve(previous_tables_metadata.size()); + std::set_difference(previous_tables_metadata.begin(), previous_tables_metadata.end(), tables_metadata.begin(), + tables_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) + { + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "{} were removed or changed its definition during scanning", + tableNameWithTypeToString(difference[0].first.database, difference[0].first.table, true)}; + return false; + } + } + + return true; } /// Make backup entries for all the definitions of all the databases found. @@ -444,14 +645,16 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() { for (const auto & [database_name, database_info] : database_infos) { + if (!database_info.create_database_query) + continue; /// We store CREATE DATABASE queries only if there was BACKUP DATABASE specified. + LOG_TRACE(log, "Adding definition of database {}", backQuoteIfNeed(database_name)); ASTPtr new_create_query = database_info.create_database_query; - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); - - String new_database_name = renaming_map.getNewDatabaseName(database_name); - auto metadata_path_in_backup = root_path_in_backup / "metadata" / (escapeForFileName(new_database_name) + ".sql"); + adjustCreateQueryForBackup(new_create_query, context->getGlobalContext(), nullptr); + renameDatabaseAndTableNameInCreateQuery(new_create_query, renaming_map, context->getGlobalContext()); + const String & metadata_path_in_backup = database_info.metadata_path_in_backup; backup_entries.emplace_back(metadata_path_in_backup, std::make_shared(serializeAST(*new_create_query))); } } @@ -459,26 +662,15 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() /// Calls IDatabase::backupTable() for all the tables found to make backup entries for tables. void BackupEntriesCollector::makeBackupEntriesForTablesDefs() { - for (const auto & [key, table_info] : table_infos) + for (auto & [table_name, table_info] : table_infos) { - LOG_TRACE(log, "Adding definition of {}table {}", (key.is_temporary ? "temporary " : ""), key.name.getFullName()); + LOG_TRACE(log, "Adding definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); ASTPtr new_create_query = table_info.create_table_query; - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); - - fs::path metadata_path_in_backup; - if (key.is_temporary) - { - auto new_name = renaming_map.getNewTemporaryTableName(key.name.table); - metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(new_name) + ".sql"); - } - else - { - auto new_name = renaming_map.getNewTableName(key.name); - metadata_path_in_backup - = root_path_in_backup / "metadata" / escapeForFileName(new_name.database) / (escapeForFileName(new_name.table) + ".sql"); - } + adjustCreateQueryForBackup(new_create_query, context->getGlobalContext(), &table_info.replicated_table_shared_id); + renameDatabaseAndTableNameInCreateQuery(new_create_query, renaming_map, context->getGlobalContext()); + const String & metadata_path_in_backup = table_info.metadata_path_in_backup; backup_entries.emplace_back(metadata_path_in_backup, std::make_shared(serializeAST(*new_create_query))); } } @@ -488,63 +680,76 @@ void BackupEntriesCollector::makeBackupEntriesForTablesData() if (backup_settings.structure_only) return; - for (const auto & [key, table_info] : table_infos) + for (const auto & [table_name, table_info] : table_infos) { - LOG_TRACE(log, "Adding data of {}table {}", (key.is_temporary ? "temporary " : ""), key.name.getFullName()); const auto & storage = table_info.storage; const auto & data_path_in_backup = table_info.data_path_in_backup; - const auto & partitions = table_info.partitions; - storage->backupData(*this, data_path_in_backup, partitions); + if (storage) + { + LOG_TRACE(log, "Adding data of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); + storage->backupData(*this, data_path_in_backup, table_info.partitions); + } + else + { + /// Storage == null means this storage exists on other replicas but it has not been created on this replica yet. + /// If this table is replicated in this case we call IBackupCoordination::addReplicatedDataPath() which will cause + /// other replicas to fill the storage's data in the backup. + /// If this table is not replicated we'll do nothing leaving the storage's data empty in the backup. + if (table_info.replicated_table_shared_id) + backup_coordination->addReplicatedDataPath(*table_info.replicated_table_shared_id, data_path_in_backup); + } } } void BackupEntriesCollector::addBackupEntry(const String & file_name, BackupEntryPtr backup_entry) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding backup entries is not allowed"); backup_entries.emplace_back(file_name, backup_entry); } +void BackupEntriesCollector::addBackupEntry(const std::pair & backup_entry) +{ + addBackupEntry(backup_entry.first, backup_entry.second); +} + void BackupEntriesCollector::addBackupEntries(const BackupEntries & backup_entries_) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding backup entries is not allowed"); insertAtEnd(backup_entries, backup_entries_); } void BackupEntriesCollector::addBackupEntries(BackupEntries && backup_entries_) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding backup entries is not allowed"); insertAtEnd(backup_entries, std::move(backup_entries_)); } -void BackupEntriesCollector::addPostCollectingTask(std::function task) +void BackupEntriesCollector::addPostTask(std::function task) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding post tasks is not allowed"); - post_collecting_tasks.push(std::move(task)); + post_tasks.push(std::move(task)); } /// Runs all the tasks added with addPostCollectingTask(). -void BackupEntriesCollector::runPostCollectingTasks() +void BackupEntriesCollector::runPostTasks() { /// Post collecting tasks can add other post collecting tasks, our code is fine with that. - while (!post_collecting_tasks.empty()) + while (!post_tasks.empty()) { - auto task = std::move(post_collecting_tasks.front()); - post_collecting_tasks.pop(); + auto task = std::move(post_tasks.front()); + post_tasks.pop(); std::move(task)(); } } -void BackupEntriesCollector::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine) +size_t BackupEntriesCollector::getAccessCounter(AccessEntityType type) { - throw Exception( - ErrorCodes::CANNOT_BACKUP_TABLE, - "Table engine {} doesn't support partitions, cannot backup table {}", - table_engine, - storage_id.getFullTableName()); + access_counters.resize(static_cast(AccessEntityType::MAX)); + return access_counters[static_cast(type)]++; } } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 1466815f3a7..9a653ee7e4d 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -19,6 +19,7 @@ class IBackupCoordination; class IDatabase; using DatabasePtr = std::shared_ptr; struct StorageID; +enum class AccessEntityType; /// Collects backup entries for all databases and tables which should be put to a backup. class BackupEntriesCollector : private boost::noncopyable @@ -27,84 +28,90 @@ public: BackupEntriesCollector(const ASTBackupQuery::Elements & backup_query_elements_, const BackupSettings & backup_settings_, std::shared_ptr backup_coordination_, - const ContextPtr & context_, - std::chrono::seconds timeout_ = std::chrono::seconds(-1) /* no timeout */); + const ContextPtr & context_); ~BackupEntriesCollector(); /// Collects backup entries and returns the result. - /// This function first generates a list of databases and then call IDatabase::backup() for each database from this list. - /// At this moment IDatabase::backup() calls IStorage::backup() and they both call addBackupEntry() to build a list of backup entries. - BackupEntries getBackupEntries(); + /// This function first generates a list of databases and then call IDatabase::getTablesForBackup() for each database from this list. + /// Then it calls IStorage::backupData() to build a list of backup entries. + BackupEntries run(); const BackupSettings & getBackupSettings() const { return backup_settings; } std::shared_ptr getBackupCoordination() const { return backup_coordination; } ContextPtr getContext() const { return context; } - /// Adds a backup entry which will be later returned by getBackupEntries(). - /// These function can be called by implementations of IStorage::backup() in inherited storage classes. + /// Adds a backup entry which will be later returned by run(). + /// These function can be called by implementations of IStorage::backupData() in inherited storage classes. void addBackupEntry(const String & file_name, BackupEntryPtr backup_entry); + void addBackupEntry(const std::pair & backup_entry); void addBackupEntries(const BackupEntries & backup_entries_); void addBackupEntries(BackupEntries && backup_entries_); - /// Adds a function which must be called after all IStorage::backup() have finished their work on all hosts. + /// Adds a function which must be called after all IStorage::backupData() have finished their work on all hosts. /// This function is designed to help making a consistent in some complex cases like /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. - void addPostCollectingTask(std::function task); + void addPostTask(std::function task); - /// Writing a backup includes a few stages: - enum class Stage - { - /// Initial stage. - kPreparing, - - /// Finding all tables and databases which we're going to put to the backup. - kFindingTables, - - /// Making temporary hard links and prepare backup entries. - kExtractingDataFromTables, - - /// Running special tasks for replicated databases or tables which can also prepare some backup entries. - kRunningPostTasks, - - /// Writing backup entries to the backup and removing temporary hard links. - kWritingBackup, - - /// An error happens during any of the stages above, the backup won't be written. - kError, - }; - static std::string_view toString(Stage stage); - - /// Throws an exception that a specified table engine doesn't support partitions. - [[noreturn]] static void throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine); + /// Returns an incremental counter used to backup access control. + size_t getAccessCounter(AccessEntityType type); private: - void setStage(Stage new_stage, const String & error_message = {}); void calculateRootPathInBackup(); - void collectDatabasesAndTablesInfo(); - void collectTableInfo(const QualifiedTableName & table_name, bool is_temporary_table, const std::optional & partitions, bool throw_if_not_found); - void collectDatabaseInfo(const String & database_name, const std::set & except_table_names, bool throw_if_not_found); - void collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names); - void checkConsistency(); + + void gatherMetadataAndCheckConsistency(); + + bool tryGatherMetadataAndCompareWithPrevious(std::optional & inconsistency_error); + + void gatherDatabasesMetadata(); + + void gatherDatabaseMetadata( + const String & database_name, + bool throw_if_database_not_found, + bool backup_create_database_query, + const std::optional & table_name, + bool throw_if_table_not_found, + const std::optional & partitions, + bool all_tables, + const std::set & except_table_names); + + void gatherTablesMetadata(); + void lockTablesForReading(); + bool compareWithPrevious(std::optional & inconsistency_error); + void makeBackupEntriesForDatabasesDefs(); void makeBackupEntriesForTablesDefs(); void makeBackupEntriesForTablesData(); - void runPostCollectingTasks(); + void runPostTasks(); + + Strings setStatus(const String & new_status, const String & message = ""); const ASTBackupQuery::Elements backup_query_elements; const BackupSettings backup_settings; std::shared_ptr backup_coordination; ContextPtr context; - std::chrono::seconds timeout; + std::chrono::milliseconds consistent_metadata_snapshot_timeout; Poco::Logger * log; - Stage current_stage = Stage::kPreparing; - std::filesystem::path root_path_in_backup; + Strings all_hosts; DDLRenamingMap renaming_map; + std::filesystem::path root_path_in_backup; struct DatabaseInfo { DatabasePtr database; ASTPtr create_database_query; + String metadata_path_in_backup; + + struct TableParams + { + bool throw_if_table_not_found = false; + std::optional partitions; + }; + + std::unordered_map tables; + + bool all_tables = false; + std::unordered_set except_table_names; }; struct TableInfo @@ -113,26 +120,22 @@ private: StoragePtr storage; TableLockHolder table_lock; ASTPtr create_table_query; + String metadata_path_in_backup; std::filesystem::path data_path_in_backup; + std::optional replicated_table_shared_id; std::optional partitions; }; - struct TableKey - { - QualifiedTableName name; - bool is_temporary = false; - bool operator ==(const TableKey & right) const; - bool operator <(const TableKey & right) const; - }; - + String current_status; + std::chrono::steady_clock::time_point consistent_metadata_snapshot_start_time; std::unordered_map database_infos; - std::map table_infos; - std::optional> previous_database_names; - std::optional> previous_table_names; - bool consistent = false; + std::unordered_map table_infos; + std::vector> previous_databases_metadata; + std::vector> previous_tables_metadata; BackupEntries backup_entries; - std::queue> post_collecting_tasks; + std::queue> post_tasks; + std::vector access_counters; }; } diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index c5de4bd7e67..9ff91050177 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -39,7 +39,7 @@ DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & e const String & new_table_name = element.new_table_name; assert(!table_name.empty()); assert(!new_table_name.empty()); - map.setNewTemporaryTableName(table_name, new_table_name); + map.setNewTableName({DatabaseCatalog::TEMPORARY_DATABASE, table_name}, {DatabaseCatalog::TEMPORARY_DATABASE, new_table_name}); break; } diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 84dc63b4f9f..635b2810941 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -166,9 +166,8 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c BackupEntries backup_entries; { - auto timeout = std::chrono::seconds{context_in_use->getConfigRef().getInt("backups.backup_prepare_timeout", -1)}; - BackupEntriesCollector backup_entries_collector{backup_query->elements, backup_settings, backup_coordination, context_in_use, timeout}; - backup_entries = backup_entries_collector.getBackupEntries(); + BackupEntriesCollector backup_entries_collector{backup_query->elements, backup_settings, backup_coordination, context_in_use}; + backup_entries = backup_entries_collector.run(); } writeBackupEntries(backup, std::move(backup_entries), backups_thread_pool); @@ -272,8 +271,8 @@ UUID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr conte String addr_database = address->default_database.empty() ? current_database : address->default_database; for (auto & element : restore_elements) element.setCurrentDatabase(addr_database); - RestorerFromBackup dummy_restorer{restore_elements, restore_settings, nullptr, backup, context_in_use, {}}; - dummy_restorer.checkAccessOnly(); + RestorerFromBackup dummy_restorer{restore_elements, restore_settings, nullptr, backup, context_in_use}; + dummy_restorer.run(RestorerFromBackup::CHECK_ACCESS_ONLY); } } @@ -325,11 +324,9 @@ UUID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr conte DataRestoreTasks data_restore_tasks; { - auto timeout = std::chrono::seconds{context_in_use->getConfigRef().getInt("backups.restore_metadata_timeout", -1)}; RestorerFromBackup restorer{restore_query->elements, restore_settings, restore_coordination, - backup, context_in_use, timeout}; - restorer.restoreMetadata(); - data_restore_tasks = restorer.getDataRestoreTasks(); + backup, context_in_use}; + data_restore_tasks = restorer.run(RestorerFromBackup::RESTORE); } restoreTablesData(std::move(data_restore_tasks), restores_thread_pool); diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp new file mode 100644 index 00000000000..8223e08f127 --- /dev/null +++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace +{ + void visitStorageSystemTableEngine(ASTStorage &, const DDLAdjustingForBackupVisitor::Data & data) + { + /// Precondition: storage.engine && storage.engine->name.starts_with("System")) + + /// If this is a definition of a system table we'll remove columns and comment because they're redundant for backups. + auto & create = data.create_query->as(); + create.reset(create.columns_list); + create.reset(create.comment); + } + + void visitStorageReplicatedTableEngine(ASTStorage & storage, const DDLAdjustingForBackupVisitor::Data & data) + { + /// Precondition: engine_name.starts_with("Replicated") && engine_name.ends_with("MergeTree") + + if (data.replicated_table_shared_id) + *data.replicated_table_shared_id = StorageReplicatedMergeTree::tryGetTableSharedIDFromCreateQuery(*data.create_query, data.global_context); + + /// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there + /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. + /// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table. + auto & create = data.create_query->as(); + auto & engine = *storage.engine; + + auto * engine_args_ast = typeid_cast(engine.arguments.get()); + if (!engine_args_ast) + return; + + auto & engine_args = engine_args_ast->children; + if (engine_args.size() < 2) + return; + + auto * zookeeper_path_ast = typeid_cast(engine_args[0].get()); + auto * replica_name_ast = typeid_cast(engine_args[1].get()); + if (zookeeper_path_ast && (zookeeper_path_ast->value.getType() == Field::Types::String) && + replica_name_ast && (replica_name_ast->value.getType() == Field::Types::String)) + { + String & zookeeper_path_arg = zookeeper_path_ast->value.get(); + String & replica_name_arg = replica_name_ast->value.get(); + if (create.uuid != UUIDHelpers::Nil) + { + String table_uuid_str = toString(create.uuid); + if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) + zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); + } + const auto & config = data.global_context->getConfigRef(); + if ((zookeeper_path_arg == StorageReplicatedMergeTree::getDefaultZooKeeperPath(config)) + && (replica_name_arg == StorageReplicatedMergeTree::getDefaultReplicaName(config)) + && ((engine_args.size() == 2) || !engine_args[2]->as())) + { + engine_args.erase(engine_args.begin(), engine_args.begin() + 2); + } + } + } + + void visitStorage(ASTStorage & storage, const DDLAdjustingForBackupVisitor::Data & data) + { + if (!storage.engine) + return; + + const String & engine_name = storage.engine->name; + if (engine_name.starts_with("System")) + visitStorageSystemTableEngine(storage, data); + else if (engine_name.starts_with("Replicated") && engine_name.ends_with("MergeTree")) + visitStorageReplicatedTableEngine(storage, data); + } + + void visitCreateQuery(ASTCreateQuery & create, const DDLAdjustingForBackupVisitor::Data & data) + { + create.uuid = UUIDHelpers::Nil; + create.to_inner_uuid = UUIDHelpers::Nil; + + if (create.storage) + visitStorage(*create.storage, data); + } +} + + +bool DDLAdjustingForBackupVisitor::needChildVisit(const ASTPtr &, const ASTPtr &) +{ + return false; +} + +void DDLAdjustingForBackupVisitor::visit(ASTPtr ast, const Data & data) +{ + if (auto * create = ast->as()) + visitCreateQuery(*create, data); +} + +void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id) +{ + if (replicated_table_shared_id) + *replicated_table_shared_id = {}; + + DDLAdjustingForBackupVisitor::Data data{ast, global_context, replicated_table_shared_id}; + DDLAdjustingForBackupVisitor::Visitor{data}.visit(ast); +} + +} diff --git a/src/Backups/DDLAdjustingForBackupVisitor.h b/src/Backups/DDLAdjustingForBackupVisitor.h new file mode 100644 index 00000000000..63353dcc000 --- /dev/null +++ b/src/Backups/DDLAdjustingForBackupVisitor.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class IAST; +using ASTPtr = std::shared_ptr; +class Context; +using ContextPtr = std::shared_ptr; + +/// Changes a create query to a form which is appropriate or suitable for saving in a backup. +/// Also extracts a replicated table's shared ID from the create query if this is a create query for a replicated table. +/// `replicated_table_shared_id` can be null if you don't need that. +void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id); + +/// Visits ASTCreateQuery and changes it to a form which is appropriate or suitable for saving in a backup. +class DDLAdjustingForBackupVisitor +{ +public: + struct Data + { + ASTPtr create_query; + ContextPtr global_context; + std::optional * replicated_table_shared_id = nullptr; + }; + + using Visitor = InDepthNodeVisitor; + + static bool needChildVisit(const ASTPtr & ast, const ASTPtr & child); + static void visit(ASTPtr ast, const Data & data); +}; + +} diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 92b7139ed5f..b4c5c7b3d88 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -13,11 +13,10 @@ class IBackupCoordination public: virtual ~IBackupCoordination() = default; - /// Sets the current stage and waits for other hosts to come to this stage too. - virtual void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) = 0; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - virtual void syncStageError(const String & current_host, const String & error_message) = 0; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + virtual void setStatus(const String & current_host, const String & new_status, const String & message) = 0; + virtual Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts) = 0; + virtual Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts, UInt64 timeout_ms) = 0; struct PartNameAndChecksum { @@ -29,21 +28,29 @@ public: /// Multiple replicas of the replicated table call this function and then the added part names can be returned by call of the function /// getReplicatedPartNames(). /// Checksums are used only to control that parts under the same names on different replicas are the same. - virtual void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, + virtual void addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) = 0; /// Returns the names of the parts which a specified replica of a replicated table should put to the backup. /// This is the same list as it was added by call of the function addReplicatedPartNames() but without duplications and without /// parts covered by another parts. - virtual Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const = 0; + virtual Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const = 0; /// Adds a data path in backup for a replicated table. /// Multiple replicas of the replicated table call this function and then all the added paths can be returned by call of the function /// getReplicatedDataPaths(). - virtual void addReplicatedDataPath(const String & table_zk_path, const String & data_path) = 0; + virtual void addReplicatedDataPath(const String & table_shared_id, const String & data_path) = 0; /// Returns all the data paths in backup added for a replicated table (see also addReplicatedDataPath()). - virtual Strings getReplicatedDataPaths(const String & table_zk_path) const = 0; + virtual Strings getReplicatedDataPaths(const String & table_shared_id) const = 0; + + /// Adds a path to access.txt file keeping access entities of a ReplicatedAccessStorage. + virtual void addReplicatedAccessPath(const String & access_zk_path, const String & file_path) = 0; + virtual Strings getReplicatedAccessPaths(const String & access_zk_path) const = 0; + + /// Sets the host id of a host storing access entities of a ReplicatedAccessStorage to backup. + virtual void setReplicatedAccessHost(const String & access_zk_path, const String & host) = 0; + virtual String getReplicatedAccessHost(const String & access_zk_path) const = 0; struct FileInfo { diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index fd9a67e1b96..ba76a6e0c99 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -13,11 +13,10 @@ class IRestoreCoordination public: virtual ~IRestoreCoordination() = default; - /// Sets the current stage and waits for other hosts to come to this stage too. - virtual void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) = 0; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - virtual void syncStageError(const String & current_host, const String & error_message) = 0; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + virtual void setStatus(const String & current_host, const String & new_status, const String & message) = 0; + virtual Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts) = 0; + virtual Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts, UInt64 timeout_ms) = 0; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. virtual bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) = 0; diff --git a/src/Backups/RestoreCoordinationDistributed.cpp b/src/Backups/RestoreCoordinationDistributed.cpp index e131ce7fe24..8cbaa01810d 100644 --- a/src/Backups/RestoreCoordinationDistributed.cpp +++ b/src/Backups/RestoreCoordinationDistributed.cpp @@ -9,7 +9,7 @@ namespace DB RestoreCoordinationDistributed::RestoreCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , stage_sync(zookeeper_path_ + "/stage", get_zookeeper_, &Poco::Logger::get("RestoreCoordination")) + , status_sync(zookeeper_path_ + "/status", get_zookeeper_, &Poco::Logger::get("RestoreCoordination")) { createRootNodes(); } @@ -26,14 +26,19 @@ void RestoreCoordinationDistributed::createRootNodes() zookeeper->createIfNotExists(zookeeper_path + "/repl_access_storages_acquired", ""); } -void RestoreCoordinationDistributed::syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) +void RestoreCoordinationDistributed::setStatus(const String & current_host, const String & new_status, const String & message) { - stage_sync.syncStage(current_host, new_stage, wait_hosts, timeout); + status_sync.set(current_host, new_status, message); } -void RestoreCoordinationDistributed::syncStageError(const String & current_host, const String & error_message) +Strings RestoreCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) { - stage_sync.syncStageError(current_host, error_message); + return status_sync.setAndWait(current_host, new_status, message, all_hosts); +} + +Strings RestoreCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) +{ + return status_sync.setAndWaitFor(current_host, new_status, message, all_hosts, timeout_ms); } bool RestoreCoordinationDistributed::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationDistributed.h b/src/Backups/RestoreCoordinationDistributed.h index 0ea5db3f062..52b961cf0ef 100644 --- a/src/Backups/RestoreCoordinationDistributed.h +++ b/src/Backups/RestoreCoordinationDistributed.h @@ -14,11 +14,10 @@ public: RestoreCoordinationDistributed(const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper); ~RestoreCoordinationDistributed() override; - /// Sets the current stage and waits for other hosts to come to this stage too. - void syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - void syncStageError(const String & current_host, const String & error_message) override; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; @@ -42,7 +41,7 @@ private: const String zookeeper_path; const zkutil::GetZooKeeper get_zookeeper; - BackupCoordinationStageSync stage_sync; + BackupCoordinationStatusSync status_sync; }; } diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index 9cecc3f90c9..a999cc13195 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -7,12 +7,18 @@ namespace DB RestoreCoordinationLocal::RestoreCoordinationLocal() = default; RestoreCoordinationLocal::~RestoreCoordinationLocal() = default; -void RestoreCoordinationLocal::syncStage(const String &, int, const Strings &, std::chrono::seconds) +void RestoreCoordinationLocal::setStatus(const String &, const String &, const String &) { } -void RestoreCoordinationLocal::syncStageError(const String &, const String &) +Strings RestoreCoordinationLocal::setStatusAndWait(const String &, const String &, const String &, const Strings &) { + return {}; +} + +Strings RestoreCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const String &, const Strings &, UInt64) +{ + return {}; } bool RestoreCoordinationLocal::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index b73f345df47..68624481a7a 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -17,11 +17,10 @@ public: RestoreCoordinationLocal(); ~RestoreCoordinationLocal() override; - /// Sets the current stage and waits for other hosts to come to this stage too. - void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - void syncStageError(const String & current_host, const String & error_message) override; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; diff --git a/src/Backups/RestoreSettings.cpp b/src/Backups/RestoreSettings.cpp index 590d39f24f8..efa1fe2cfb8 100644 --- a/src/Backups/RestoreSettings.cpp +++ b/src/Backups/RestoreSettings.cpp @@ -74,7 +74,7 @@ namespace { case RestoreTableCreationMode::kCreate: return Field{true}; case RestoreTableCreationMode::kMustExist: return Field{false}; - case RestoreTableCreationMode::kCreateIfNotExists: return Field{"if not exists"}; + case RestoreTableCreationMode::kCreateIfNotExists: return Field{"if-not-exists"}; } throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected value of enum RestoreTableCreationMode: {}", static_cast(value)); } @@ -131,12 +131,14 @@ namespace switch (value) { case RestoreAccessCreationMode::kCreate: return Field{true}; - case RestoreAccessCreationMode::kCreateIfNotExists: return Field{"if not exists"}; + case RestoreAccessCreationMode::kCreateIfNotExists: return Field{"if-not-exists"}; case RestoreAccessCreationMode::kReplace: return Field{"replace"}; } throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected value of enum RestoreAccessCreationMode: {}", static_cast(value)); } }; + + using SettingFieldRestoreUDFCreationMode = SettingFieldRestoreAccessCreationMode; } /// List of restore settings except base_backup_name and cluster_host_ids. @@ -155,6 +157,7 @@ namespace M(Bool, allow_non_empty_tables) \ M(RestoreAccessCreationMode, create_access) \ M(Bool, allow_unresolved_access_dependencies) \ + M(RestoreUDFCreationMode, create_function) \ M(Bool, internal) \ M(String, host_id) \ M(String, coordination_zk_path) diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h index 5e941b79508..1bc5d867a37 100644 --- a/src/Backups/RestoreSettings.h +++ b/src/Backups/RestoreSettings.h @@ -36,6 +36,8 @@ enum class RestoreAccessCreationMode kReplace, }; +using RestoreUDFCreationMode = RestoreAccessCreationMode; + /// Settings specified in the "SETTINGS" clause of a RESTORE query. struct RestoreSettings { @@ -99,6 +101,9 @@ struct RestoreSettings /// For example, if an user has a profile assigned and that profile is not in the backup and doesn't exist locally. bool allow_unresolved_access_dependencies = false; + /// How the RESTORE command will handle if a user-defined function which it's going to restore already exists. + RestoreUDFCreationMode create_function = RestoreUDFCreationMode::kCreateIfNotExists; + /// Internal, should not be specified by user. bool internal = false; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 14f5b7f48f0..5b211bc50a8 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -1,9 +1,11 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -39,54 +41,58 @@ namespace ErrorCodes namespace { - constexpr const std::string_view sql_ext = ".sql"; + /// Finding databases and tables in the backup which we're going to restore. + constexpr const char * kFindingTablesInBackupStatus = "finding tables in backup"; - String tryGetTableEngine(const IAST & ast) + /// Creating databases or finding them and checking their definitions. + constexpr const char * kCreatingDatabasesStatus = "creating databases"; + + /// Creating tables or finding them and checking their definition. + constexpr const char * kCreatingTablesStatus = "creating tables"; + + /// Inserting restored data to tables. + constexpr const char * kInsertingDataToTablesStatus = "inserting data to tables"; + + /// Error status. + constexpr const char * kErrorStatus = BackupCoordinationStatusSync::kErrorStatus; + + /// Uppercases the first character of a passed string. + String toUpperFirst(const String & str) { - const ASTCreateQuery * create = ast.as(); - if (!create) - return {}; - if (!create->storage || !create->storage->engine) - return {}; - return create->storage->engine->name; + String res = str; + res[0] = std::toupper(res[0]); + return res; } - bool hasSystemTableEngine(const IAST & ast) + /// Outputs "table " or "temporary table " + String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_upper) { - return tryGetTableEngine(ast).starts_with("System"); + String str; + if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) + str = fmt::format("temporary table {}", backQuoteIfNeed(table_name)); + else + str = fmt::format("table {}.{}", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + if (first_upper) + str[0] = std::toupper(str[0]); + return str; } - bool hasSystemAccessTableEngine(const IAST & ast) + /// Whether a specified name corresponds one of the tables backuping ACL. + bool isSystemAccessTableName(const QualifiedTableName & table_name) { - String engine_name = tryGetTableEngine(ast); - return (engine_name == "SystemUsers") || (engine_name == "SystemRoles") || (engine_name == "SystemSettingsProfiles") - || (engine_name == "SystemRowPolicies") || (engine_name == "SystemQuotas"); + if (table_name.database != DatabaseCatalog::SYSTEM_DATABASE) + return false; + + return (table_name.table == "users") || (table_name.table == "roles") || (table_name.table == "settings_profiles") + || (table_name.table == "row_policies") || (table_name.table == "quotas"); } -} -bool RestorerFromBackup::TableKey::operator ==(const TableKey & right) const -{ - return (name == right.name) && (is_temporary == right.is_temporary); -} - -bool RestorerFromBackup::TableKey::operator <(const TableKey & right) const -{ - return (name < right.name) || ((name == right.name) && (is_temporary < right.is_temporary)); -} - -std::string_view RestorerFromBackup::toString(Stage stage) -{ - switch (stage) + /// Whether a specified name corresponds one of the tables backuping ACL. + bool isSystemFunctionsTableName(const QualifiedTableName & table_name) { - case Stage::kPreparing: return "Preparing"; - case Stage::kFindingTablesInBackup: return "Finding tables in backup"; - case Stage::kCreatingDatabases: return "Creating databases"; - case Stage::kCreatingTables: return "Creating tables"; - case Stage::kInsertingDataToTables: return "Inserting data to tables"; - case Stage::kError: return "Error"; + return (table_name.database == DatabaseCatalog::SYSTEM_DATABASE) && (table_name.table == "functions"); } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown restore stage: {}", static_cast(stage)); -} + } RestorerFromBackup::RestorerFromBackup( @@ -94,71 +100,66 @@ RestorerFromBackup::RestorerFromBackup( const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_, const BackupPtr & backup_, - const ContextMutablePtr & context_, - std::chrono::seconds timeout_) + const ContextMutablePtr & context_) : restore_query_elements(restore_query_elements_) , restore_settings(restore_settings_) , restore_coordination(restore_coordination_) , backup(backup_) , context(context_) - , timeout(timeout_) + , create_table_timeout(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) , log(&Poco::Logger::get("RestorerFromBackup")) { } RestorerFromBackup::~RestorerFromBackup() = default; -void RestorerFromBackup::restoreMetadata() -{ - run(/* only_check_access= */ false); -} - -void RestorerFromBackup::checkAccessOnly() -{ - run(/* only_check_access= */ true); -} - -void RestorerFromBackup::run(bool only_check_access) +RestorerFromBackup::DataRestoreTasks RestorerFromBackup::run(Mode mode) { try { - /// restoreMetadata() must not be called multiple times. - if (current_stage != Stage::kPreparing) + /// run() can be called onle once. + if (!current_status.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Already restoring"); - /// Calculate the root path in the backup for restoring, it's either empty or has the format "shards//replicas//". - findRootPathsInBackup(); + /// Find other hosts working along with us to execute this ON CLUSTER query. + all_hosts = BackupSettings::Util::filterHostIDs( + restore_settings.cluster_host_ids, restore_settings.shard_num, restore_settings.replica_num); /// Do renaming in the create queries according to the renaming config. renaming_map = makeRenamingMapFromBackupQuery(restore_query_elements); + /// Calculate the root path in the backup for restoring, it's either empty or has the format "shards//replicas//". + findRootPathsInBackup(); + /// Find all the databases and tables which we will read from the backup. - setStage(Stage::kFindingTablesInBackup); - collectDatabaseAndTableInfos(); + setStatus(kFindingTablesInBackupStatus); + findDatabasesAndTablesInBackup(); /// Check access rights. - checkAccessForCollectedInfos(); - if (only_check_access) - return; + checkAccessForObjectsFoundInBackup(); + + if (mode == Mode::CHECK_ACCESS_ONLY) + return {}; /// Create databases using the create queries read from the backup. - setStage(Stage::kCreatingDatabases); + setStatus(kCreatingDatabasesStatus); createDatabases(); /// Create tables using the create queries read from the backup. - setStage(Stage::kCreatingTables); + setStatus(kCreatingTablesStatus); createTables(); /// All what's left is to insert data to tables. /// No more data restoring tasks are allowed after this point. - setStage(Stage::kInsertingDataToTables); + setStatus(kInsertingDataToTablesStatus); + return getDataRestoreTasks(); } catch (...) { try { /// Other hosts should know that we've encountered an error. - setStage(Stage::kError, getCurrentExceptionMessage(false)); + setStatus(kErrorStatus, getCurrentExceptionMessage(false)); } catch (...) { @@ -167,59 +168,20 @@ void RestorerFromBackup::run(bool only_check_access) } } - -RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() +void RestorerFromBackup::setStatus(const String & new_status, const String & message) { - if (current_stage != Stage::kInsertingDataToTables) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Metadata wasn't restored"); - - if (data_restore_tasks.empty() && !access_restore_task) - return {}; - - LOG_TRACE(log, "Will insert data to tables"); - - /// Storages and table locks must exist while we're executing data restoring tasks. - auto storages = std::make_shared>(); - auto table_locks = std::make_shared>(); - storages->reserve(table_infos.size()); - table_locks->reserve(table_infos.size()); - for (const auto & table_info : table_infos | boost::adaptors::map_values) + if (new_status == kErrorStatus) { - storages->push_back(table_info.storage); - table_locks->push_back(table_info.table_lock); - } - - DataRestoreTasks res_tasks; - for (const auto & task : data_restore_tasks) - res_tasks.push_back([task, storages, table_locks] { task(); }); - - if (access_restore_task) - res_tasks.push_back([task = access_restore_task, access_control = &context->getAccessControl()] { task->restore(*access_control); }); - - return res_tasks; -} - -void RestorerFromBackup::setStage(Stage new_stage, const String & error_message) -{ - if (new_stage == Stage::kError) - LOG_ERROR(log, "{} failed with error: {}", toString(current_stage), error_message); - else - LOG_TRACE(log, "{}", toString(new_stage)); - - current_stage = new_stage; - - if (!restore_coordination) - return; - - if (new_stage == Stage::kError) - { - restore_coordination->syncStageError(restore_settings.host_id, error_message); + LOG_ERROR(log, "{} failed with {}", toUpperFirst(current_status), message); + if (restore_coordination) + restore_coordination->setStatus(restore_settings.host_id, new_status, message); } else { - auto all_hosts - = BackupSettings::Util::filterHostIDs(restore_settings.cluster_host_ids, restore_settings.shard_num, restore_settings.replica_num); - restore_coordination->syncStage(restore_settings.host_id, static_cast(new_stage), all_hosts, timeout); + LOG_TRACE(log, "{}", toUpperFirst(new_status)); + current_status = new_status; + if (restore_coordination) + restore_coordination->setStatusAndWait(restore_settings.host_id, new_status, message, all_hosts); } } @@ -302,7 +264,7 @@ void RestorerFromBackup::findRootPathsInBackup() ", ")); } -void RestorerFromBackup::collectDatabaseAndTableInfos() +void RestorerFromBackup::findDatabasesAndTablesInBackup() { database_infos.clear(); table_infos.clear(); @@ -312,22 +274,22 @@ void RestorerFromBackup::collectDatabaseAndTableInfos() { case ASTBackupQuery::ElementType::TABLE: { - collectTableInfo({element.database_name, element.table_name}, false, element.partitions); + findTableInBackup({element.database_name, element.table_name}, element.partitions); break; } case ASTBackupQuery::ElementType::TEMPORARY_TABLE: { - collectTableInfo({element.database_name, element.table_name}, true, element.partitions); + findTableInBackup({DatabaseCatalog::TEMPORARY_DATABASE, element.table_name}, element.partitions); break; } case ASTBackupQuery::ElementType::DATABASE: { - collectDatabaseInfo(element.database_name, element.except_tables, /* throw_if_no_database_metadata_in_backup= */ true); + findDatabaseInBackup(element.database_name, element.except_tables); break; } case ASTBackupQuery::ElementType::ALL: { - collectAllDatabasesInfo(element.except_databases, element.except_tables); + findEverythingInBackup(element.except_databases, element.except_tables); break; } } @@ -336,9 +298,9 @@ void RestorerFromBackup::collectDatabaseAndTableInfos() LOG_INFO(log, "Will restore {} databases and {} tables", database_infos.size(), table_infos.size()); } -void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_in_backup, bool is_temporary_table, const std::optional & partitions) +void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional & partitions) { - String database_name_in_backup = is_temporary_table ? DatabaseCatalog::TEMPORARY_DATABASE : table_name_in_backup.database; + bool is_temporary_table = (table_name_in_backup.database == DatabaseCatalog::TEMPORARY_DATABASE); std::optional metadata_path; std::optional root_path_in_use; @@ -365,21 +327,20 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_ } if (!metadata_path) - throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Table {} not found in backup", table_name_in_backup.getFullName()); + throw Exception( + ErrorCodes::BACKUP_ENTRY_NOT_FOUND, + "{} not found in backup", + tableNameWithTypeToString(table_name_in_backup.database, table_name_in_backup.table, true)); - TableKey table_key; fs::path data_path_in_backup; if (is_temporary_table) { data_path_in_backup = *root_path_in_use / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup.table); - table_key.name.table = renaming_map.getNewTemporaryTableName(table_name_in_backup.table); - table_key.is_temporary = true; } else { data_path_in_backup = *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); - table_key.name = renaming_map.getNewTableName(table_name_in_backup); } auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer(); @@ -388,27 +349,30 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_ read_buffer.reset(); ParserCreateQuery create_parser; ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, create_table_query); + renameDatabaseAndTableNameInCreateQuery(create_table_query, renaming_map, context->getGlobalContext()); - if (auto it = table_infos.find(table_key); it != table_infos.end()) + QualifiedTableName table_name = renaming_map.getNewTableName(table_name_in_backup); + + if (auto it = table_infos.find(table_name); it != table_infos.end()) { const TableInfo & table_info = it->second; if (table_info.create_table_query && (serializeAST(*table_info.create_table_query) != serializeAST(*create_table_query))) { throw Exception( ErrorCodes::CANNOT_RESTORE_TABLE, - "Extracted two different create queries for the same {}table {}: {} and {}", - (is_temporary_table ? "temporary " : ""), - table_key.name.getFullName(), + "Extracted two different create queries for the same {}: {} and {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), serializeAST(*table_info.create_table_query), serializeAST(*create_table_query)); } } - TableInfo & res_table_info = table_infos[table_key]; + TableInfo & res_table_info = table_infos[table_name]; res_table_info.create_table_query = create_table_query; + res_table_info.is_predefined_table = DatabaseCatalog::instance().isPredefinedTable(StorageID{table_name.database, table_name.table}); + res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_name, create_table_query); + res_table_info.has_data = backup->hasFiles(data_path_in_backup); res_table_info.data_path_in_backup = data_path_in_backup; - res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_key.name, create_table_query); if (partitions) { @@ -417,35 +381,45 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_ insertAtEnd(*res_table_info.partitions, *partitions); } - if (hasSystemAccessTableEngine(*create_table_query)) + if (!restore_settings.structure_only && isSystemAccessTableName(table_name)) { - if (!access_restore_task) - access_restore_task = std::make_shared(backup, restore_settings, restore_coordination); - access_restore_task->addDataPath(data_path_in_backup); + if (!access_restorer) + access_restorer = std::make_unique(backup, restore_settings); + access_restorer->addDataPath(data_path_in_backup, table_name); } } -void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_backup, const std::set & except_table_names, bool throw_if_no_database_metadata_in_backup) +void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_backup, const std::set & except_table_names) { std::optional metadata_path; std::unordered_set table_names_in_backup; for (const auto & root_path_in_backup : root_paths_in_backup) { - fs::path try_metadata_path = root_path_in_backup / "metadata" / (escapeForFileName(database_name_in_backup) + ".sql"); - if (!metadata_path && backup->fileExists(try_metadata_path)) + fs::path try_metadata_path, try_tables_metadata_path; + if (database_name_in_backup == DatabaseCatalog::TEMPORARY_DATABASE) + { + try_tables_metadata_path = root_path_in_backup / "temporary_tables" / "metadata"; + } + else + { + try_metadata_path = root_path_in_backup / "metadata" / (escapeForFileName(database_name_in_backup) + ".sql"); + try_tables_metadata_path = root_path_in_backup / "metadata" / escapeForFileName(database_name_in_backup); + } + + if (!metadata_path && !try_metadata_path.empty() && backup->fileExists(try_metadata_path)) metadata_path = try_metadata_path; - Strings file_names = backup->listFiles(root_path_in_backup / "metadata" / escapeForFileName(database_name_in_backup)); + Strings file_names = backup->listFiles(try_tables_metadata_path); for (const String & file_name : file_names) { - if (!file_name.ends_with(sql_ext)) + if (!file_name.ends_with(".sql")) continue; - String file_name_without_ext = file_name.substr(0, file_name.length() - sql_ext.length()); + String file_name_without_ext = file_name.substr(0, file_name.length() - strlen(".sql")); table_names_in_backup.insert(unescapeForFileName(file_name_without_ext)); } } - if (!metadata_path && throw_if_no_database_metadata_in_backup) + if (!metadata_path && table_names_in_backup.empty()) throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Database {} not found in backup", backQuoteIfNeed(database_name_in_backup)); if (metadata_path) @@ -456,7 +430,7 @@ void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_bac read_buffer.reset(); ParserCreateQuery create_parser; ASTPtr create_database_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, create_database_query); + renameDatabaseAndTableNameInCreateQuery(create_database_query, renaming_map, context->getGlobalContext()); String database_name = renaming_map.getNewDatabaseName(database_name_in_backup); DatabaseInfo & database_info = database_infos[database_name]; @@ -472,6 +446,7 @@ void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_bac } database_info.create_database_query = create_database_query; + database_info.is_predefined_database = DatabaseCatalog::isPredefinedDatabase(database_name); } for (const String & table_name_in_backup : table_names_in_backup) @@ -479,33 +454,26 @@ void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_bac if (except_table_names.contains({database_name_in_backup, table_name_in_backup})) continue; - collectTableInfo({database_name_in_backup, table_name_in_backup}, /* is_temporary_table= */ false, /* partitions= */ {}); + findTableInBackup({database_name_in_backup, table_name_in_backup}, /* partitions= */ {}); } } -void RestorerFromBackup::collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names) +void RestorerFromBackup::findEverythingInBackup(const std::set & except_database_names, const std::set & except_table_names) { std::unordered_set database_names_in_backup; - std::unordered_set temporary_table_names_in_backup; for (const auto & root_path_in_backup : root_paths_in_backup) { Strings file_names = backup->listFiles(root_path_in_backup / "metadata"); for (String & file_name : file_names) { - if (file_name.ends_with(sql_ext)) - file_name.resize(file_name.length() - sql_ext.length()); + if (file_name.ends_with(".sql")) + file_name.resize(file_name.length() - strlen(".sql")); database_names_in_backup.emplace(unescapeForFileName(file_name)); } - file_names = backup->listFiles(root_path_in_backup / "temporary_tables" / "metadata"); - for (String & file_name : file_names) - { - if (!file_name.ends_with(sql_ext)) - continue; - file_name.resize(file_name.length() - sql_ext.length()); - temporary_table_names_in_backup.emplace(unescapeForFileName(file_name)); - } + if (backup->hasFiles(root_path_in_backup / "temporary_tables" / "metadata")) + database_names_in_backup.emplace(DatabaseCatalog::TEMPORARY_DATABASE); } for (const String & database_name_in_backup : database_names_in_backup) @@ -513,19 +481,16 @@ void RestorerFromBackup::collectAllDatabasesInfo(const std::set & except if (except_database_names.contains(database_name_in_backup)) continue; - collectDatabaseInfo(database_name_in_backup, except_table_names, /* throw_if_no_database_metadata_in_backup= */ false); + findDatabaseInBackup(database_name_in_backup, except_table_names); } - - for (const String & temporary_table_name_in_backup : temporary_table_names_in_backup) - collectTableInfo({"", temporary_table_name_in_backup}, /* is_temporary_table= */ true, /* partitions= */ {}); } -void RestorerFromBackup::checkAccessForCollectedInfos() const +void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const { AccessRightsElements required_access; - for (const auto & database_name : database_infos | boost::adaptors::map_keys) + for (const auto & [database_name, database_info] : database_infos) { - if (DatabaseCatalog::isPredefinedDatabaseName(database_name)) + if (database_info.is_predefined_database) continue; AccessFlags flags; @@ -541,10 +506,20 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const for (const auto & [table_name, table_info] : table_infos) { - if (hasSystemTableEngine(*table_info.create_table_query)) + if (table_info.is_predefined_table) + { + if (isSystemFunctionsTableName(table_name)) + { + /// CREATE_FUNCTION privilege is required to restore the "system.functions" table. + if (!restore_settings.structure_only && table_info.has_data) + required_access.emplace_back(AccessType::CREATE_FUNCTION); + } + /// Privileges required to restore ACL system tables are checked separately + /// (see access_restore_task->getRequiredAccess() below). continue; + } - if (table_name.is_temporary) + if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) { if (restore_settings.create_table != RestoreTableCreationMode::kMustExist) required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); @@ -564,8 +539,7 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const flags |= AccessType::CREATE_TABLE; } - if (!restore_settings.structure_only && !create.is_dictionary && !create.is_ordinary_view - && backup->hasFiles(table_info.data_path_in_backup)) + if (!restore_settings.structure_only && table_info.has_data) { flags |= AccessType::INSERT; } @@ -578,11 +552,11 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const flags = AccessType::SHOW_TABLES; } - required_access.emplace_back(flags, table_name.name.database, table_name.name.table); + required_access.emplace_back(flags, table_name.database, table_name.table); } - if (access_restore_task) - insertAtEnd(required_access, access_restore_task->getRequiredAccess()); + if (access_restorer) + insertAtEnd(required_access, access_restorer->getRequiredAccess()); /// We convert to AccessRights and back to check access rights in a predictable way /// (some elements could be duplicated or not sorted). @@ -597,7 +571,7 @@ void RestorerFromBackup::createDatabases() for (const auto & [database_name, database_info] : database_infos) { bool need_create_database = (restore_settings.create_database != RestoreDatabaseCreationMode::kMustExist); - if (need_create_database && DatabaseCatalog::isPredefinedDatabaseName(database_name)) + if (database_info.is_predefined_database) need_create_database = false; /// Predefined databases always exist. if (need_create_database) @@ -610,15 +584,18 @@ void RestorerFromBackup::createDatabases() create_database_query->as().if_not_exists = true; } LOG_TRACE(log, "Creating database {}: {}", backQuoteIfNeed(database_name), serializeAST(*create_database_query)); - executeCreateQuery(create_database_query); + InterpreterCreateQuery interpreter{create_database_query, context}; + interpreter.setInternal(true); + interpreter.execute(); } DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); - if (!restore_settings.allow_different_database_def) + if (!restore_settings.allow_different_database_def && !database_info.is_predefined_database) { /// Check that the database's definition is the same as expected. - ASTPtr create_database_query = database->getCreateDatabaseQueryForBackup(); + ASTPtr create_database_query = database->getCreateDatabaseQuery(); + adjustCreateQueryForBackup(create_database_query, context->getGlobalContext(), nullptr); ASTPtr expected_create_query = database_info.create_database_query; if (serializeAST(*create_database_query) != serializeAST(*expected_create_query)) { @@ -643,62 +620,62 @@ void RestorerFromBackup::createTables() if (tables_to_create.empty()) break; /// We've already created all the tables. - for (const auto & table_key : tables_to_create) + for (const auto & table_name : tables_to_create) { - auto & table_info = table_infos.at(table_key); + auto & table_info = table_infos.at(table_name); - DatabasePtr database; - if (table_key.is_temporary) - database = DatabaseCatalog::instance().getDatabaseForTemporaryTables(); - else - database = DatabaseCatalog::instance().getDatabase(table_key.name.database); + DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_name.database); bool need_create_table = (restore_settings.create_table != RestoreTableCreationMode::kMustExist); - if (need_create_table && hasSystemTableEngine(*table_info.create_table_query)) - need_create_table = false; /// Tables with System* table engine already exist or can't be created by SQL anyway. + if (table_info.is_predefined_table) + need_create_table = false; /// Predefined tables always exist. if (need_create_table) { - /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some - /// database-specific things). auto create_table_query = table_info.create_table_query; if (restore_settings.create_table == RestoreTableCreationMode::kCreateIfNotExists) { create_table_query = create_table_query->clone(); create_table_query->as().if_not_exists = true; } + LOG_TRACE( log, - "Creating {}table {}: {}", - (table_key.is_temporary ? "temporary " : ""), - table_key.name.getFullName(), + "Creating {}: {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), serializeAST(*create_table_query)); - database->createTableRestoredFromBackup(create_table_query, *this); + /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some + /// database-specific things). + database->createTableRestoredFromBackup( + create_table_query, + context, + restore_coordination, + std::chrono::duration_cast(create_table_timeout).count()); } table_info.created = true; - auto resolved_id = table_key.is_temporary - ? context->resolveStorageID(StorageID{"", table_key.name.table}, Context::ResolveExternal) - : context->resolveStorageID(StorageID{table_key.name.database, table_key.name.table}, Context::ResolveGlobal); + auto resolved_id = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) + ? context->resolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal) + : context->resolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal); auto storage = database->getTable(resolved_id.table_name, context); table_info.storage = storage; table_info.table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - if (!restore_settings.allow_different_table_def) + if (!restore_settings.allow_different_table_def && !table_info.is_predefined_table) { - ASTPtr create_table_query = storage->getCreateQueryForBackup(context, nullptr); + ASTPtr create_table_query = database->getCreateTableQuery(resolved_id.table_name, context); + adjustCreateQueryForBackup(create_table_query, context->getGlobalContext(), nullptr); ASTPtr expected_create_query = table_info.create_table_query; if (serializeAST(*create_table_query) != serializeAST(*expected_create_query)) { throw Exception( ErrorCodes::CANNOT_RESTORE_TABLE, - "The {}table {} has a different definition: {} " + "{} has a different definition: {} " "comparing to its definition in the backup: {}", - (table_key.is_temporary ? "temporary " : ""), - table_key.name.getFullName(), + tableNameWithTypeToString(table_name.database, table_name.table, true), serializeAST(*create_table_query), serializeAST(*expected_create_query)); } @@ -708,6 +685,15 @@ void RestorerFromBackup::createTables() { const auto & data_path_in_backup = table_info.data_path_in_backup; const auto & partitions = table_info.partitions; + if (partitions && !storage->supportsBackupPartition()) + { + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Table engine {} doesn't support partitions, cannot restore {}", + storage->getName(), + tableNameWithTypeToString(table_name.database, table_name.table, false)); + } + storage->restoreDataFromBackup(*this, data_path_in_backup, partitions); } } @@ -715,9 +701,9 @@ void RestorerFromBackup::createTables() } /// Returns the list of tables without dependencies or those which dependencies have been created before. -std::vector RestorerFromBackup::findTablesWithoutDependencies() const +std::vector RestorerFromBackup::findTablesWithoutDependencies() const { - std::vector tables_without_dependencies; + std::vector tables_without_dependencies; bool all_tables_created = true; for (const auto & [key, table_info] : table_infos) @@ -732,7 +718,7 @@ std::vector RestorerFromBackup::findTablesWithoutD bool all_dependencies_met = true; for (const auto & dependency : table_info.dependencies) { - auto it = table_infos.find(TableKey{dependency, false}); + auto it = table_infos.find(dependency); if ((it != table_infos.end()) && !it->second.created) { all_dependencies_met = false; @@ -751,7 +737,7 @@ std::vector RestorerFromBackup::findTablesWithoutD return {}; /// Cyclic dependency? We'll try to create those tables anyway but probably it's going to fail. - std::vector tables_with_cyclic_dependencies; + std::vector tables_with_cyclic_dependencies; for (const auto & [key, table_info] : table_infos) { if (!table_info.created) @@ -764,7 +750,7 @@ std::vector RestorerFromBackup::findTablesWithoutD "Some tables have cyclic dependency from each other: {}", boost::algorithm::join( tables_with_cyclic_dependencies - | boost::adaptors::transformed([](const TableKey & key) -> String { return key.name.getFullName(); }), + | boost::adaptors::transformed([](const QualifiedTableName & table_name) -> String { return table_name.getFullName(); }), ", ")); return tables_with_cyclic_dependencies; @@ -772,38 +758,52 @@ std::vector RestorerFromBackup::findTablesWithoutD void RestorerFromBackup::addDataRestoreTask(DataRestoreTask && new_task) { - if (current_stage == Stage::kInsertingDataToTables) + if (current_status == kInsertingDataToTablesStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding data-restoring tasks is not allowed"); data_restore_tasks.push_back(std::move(new_task)); } void RestorerFromBackup::addDataRestoreTasks(DataRestoreTasks && new_tasks) { - if (current_stage == Stage::kInsertingDataToTables) + if (current_status == kInsertingDataToTablesStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding data-restoring tasks is not allowed"); insertAtEnd(data_restore_tasks, std::move(new_tasks)); } -void RestorerFromBackup::checkPathInBackupToRestoreAccess(const String & path) +RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() { - if (!access_restore_task || !access_restore_task->hasDataPath(path)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Path to restore access was not added"); + if (data_restore_tasks.empty()) + return {}; + + LOG_TRACE(log, "Will insert data to tables"); + + /// Storages and table locks must exist while we're executing data restoring tasks. + auto storages = std::make_shared>(); + auto table_locks = std::make_shared>(); + storages->reserve(table_infos.size()); + table_locks->reserve(table_infos.size()); + for (const auto & table_info : table_infos | boost::adaptors::map_values) + { + storages->push_back(table_info.storage); + table_locks->push_back(table_info.table_lock); + } + + DataRestoreTasks res_tasks; + for (const auto & task : data_restore_tasks) + res_tasks.push_back([task, storages, table_locks] { task(); }); + + return res_tasks; } -void RestorerFromBackup::executeCreateQuery(const ASTPtr & create_query) const +std::vector> RestorerFromBackup::getAccessEntitiesToRestore() { - InterpreterCreateQuery interpreter{create_query, context}; - interpreter.setInternal(true); - interpreter.execute(); -} + if (!access_restorer || access_restored) + return {}; -void RestorerFromBackup::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine) -{ - throw Exception( - ErrorCodes::CANNOT_RESTORE_TABLE, - "Table engine {} doesn't support partitions, cannot table {}", - table_engine, - storage_id.getFullTableName()); + /// getAccessEntitiesToRestore() will return entities only when called first time (we don't want to restore the same entities again). + access_restored = true; + + return access_restorer->getAccessEntities(context->getAccessControl()); } void RestorerFromBackup::throwTableIsNotEmpty(const StorageID & storage_id) diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 65139e0b946..ae2f0c76832 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -15,7 +15,9 @@ class IBackup; using BackupPtr = std::shared_ptr; class IRestoreCoordination; struct StorageID; -class AccessRestoreTask; +class AccessRestorerFromBackup; +struct IAccessEntity; +using AccessEntityPtr = std::shared_ptr; /// Restores the definition of databases and tables and prepares tasks to restore the data of the tables. class RestorerFromBackup : private boost::noncopyable @@ -26,63 +28,38 @@ public: const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_, const BackupPtr & backup_, - const ContextMutablePtr & context_, - std::chrono::seconds timeout_); + const ContextMutablePtr & context_); ~RestorerFromBackup(); - /// Restores the definition of databases and tables and prepares tasks to restore the data of the tables. - /// restoreMetadata() checks access rights internally so checkAccessRightsOnly() shouldn't be called first. - void restoreMetadata(); + enum Mode + { + /// Restores databases and tables. + RESTORE, - /// Only checks access rights without restoring anything. - void checkAccessOnly(); + /// Only checks access rights without restoring anything. + CHECK_ACCESS_ONLY + }; using DataRestoreTask = std::function; using DataRestoreTasks = std::vector; - DataRestoreTasks getDataRestoreTasks(); + + /// Restores the metadata of databases and tables and returns tasks to restore the data of tables. + DataRestoreTasks run(Mode mode); BackupPtr getBackup() const { return backup; } const RestoreSettings & getRestoreSettings() const { return restore_settings; } bool isNonEmptyTableAllowed() const { return getRestoreSettings().allow_non_empty_tables; } std::shared_ptr getRestoreCoordination() const { return restore_coordination; } - std::chrono::seconds getTimeout() const { return timeout; } ContextMutablePtr getContext() const { return context; } - void executeCreateQuery(const ASTPtr & create_query) const; /// Adds a data restore task which will be later returned by getDataRestoreTasks(). /// This function can be called by implementations of IStorage::restoreFromBackup() in inherited storage classes. void addDataRestoreTask(DataRestoreTask && new_task); void addDataRestoreTasks(DataRestoreTasks && new_tasks); - /// Adds a new data path to restore access control. - void checkPathInBackupToRestoreAccess(const String & path); - - /// Reading a backup includes a few stages: - enum class Stage - { - /// Initial stage. - kPreparing, - - /// Finding databases and tables in the backup which we're going to restore. - kFindingTablesInBackup, - - /// Creating databases or finding them and checking their definitions. - kCreatingDatabases, - - /// Creating tables or finding them and checking their definition. - kCreatingTables, - - /// Inserting restored data to tables. - kInsertingDataToTables, - - /// An error happens during any of the stages above, the backup is not restored properly. - kError = -1, - }; - static std::string_view toString(Stage stage); - - /// Throws an exception that a specified table engine doesn't support partitions. - [[noreturn]] static void throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine); + /// Returns the list of access entities to restore. + std::vector> getAccessEntitiesToRestore(); /// Throws an exception that a specified table is already non-empty. [[noreturn]] static void throwTableIsNotEmpty(const StorageID & storage_id); @@ -93,54 +70,56 @@ private: std::shared_ptr restore_coordination; BackupPtr backup; ContextMutablePtr context; - std::chrono::seconds timeout; + std::chrono::milliseconds create_table_timeout; Poco::Logger * log; - Stage current_stage = Stage::kPreparing; - std::vector root_paths_in_backup; + Strings all_hosts; DDLRenamingMap renaming_map; + std::vector root_paths_in_backup; - void run(bool only_check_access); - void setStage(Stage new_stage, const String & error_message = {}); void findRootPathsInBackup(); - void collectDatabaseAndTableInfos(); - void collectTableInfo(const QualifiedTableName & table_name_in_backup, bool is_temporary_table, const std::optional & partitions); - void collectDatabaseInfo(const String & database_name_in_backup, const std::set & except_table_names, bool throw_if_no_database_metadata_in_backup); - void collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names); - void checkAccessForCollectedInfos() const; + + void findDatabasesAndTablesInBackup(); + void findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional & partitions); + void findDatabaseInBackup(const String & database_name_in_backup, const std::set & except_table_names); + void findEverythingInBackup(const std::set & except_database_names, const std::set & except_table_names); + + void checkAccessForObjectsFoundInBackup() const; + void createDatabases(); void createTables(); + DataRestoreTasks getDataRestoreTasks(); + + void setStatus(const String & new_status, const String & message = ""); + struct DatabaseInfo { ASTPtr create_database_query; + bool is_predefined_database = false; }; struct TableInfo { ASTPtr create_table_query; - std::optional partitions; - std::filesystem::path data_path_in_backup; + bool is_predefined_table = false; std::unordered_set dependencies; + bool has_data = false; + std::filesystem::path data_path_in_backup; + std::optional partitions; bool created = false; StoragePtr storage; TableLockHolder table_lock; }; - struct TableKey - { - QualifiedTableName name; - bool is_temporary = false; - bool operator ==(const TableKey & right) const; - bool operator <(const TableKey & right) const; - }; - - std::vector findTablesWithoutDependencies() const; + std::vector findTablesWithoutDependencies() const; + String current_status; std::unordered_map database_infos; - std::map table_infos; + std::map table_infos; std::vector data_restore_tasks; - std::shared_ptr access_restore_task; + std::unique_ptr access_restorer; + bool access_restored = false; }; } diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp index 6350d7b0e61..2274762a9f5 100644 --- a/src/Columns/ColumnFunction.cpp +++ b/src/Columns/ColumnFunction.cpp @@ -250,9 +250,12 @@ ColumnWithTypeAndName ColumnFunction::reduce() const "arguments but " + toString(captured) + " columns were captured.", ErrorCodes::LOGICAL_ERROR); ColumnsWithTypeAndName columns = captured_columns; - if (is_short_circuit_argument) + IFunction::ShortCircuitSettings settings; + /// Arguments of lazy executed function can also be lazy executed. + /// But we shouldn't execute arguments if this function is short circuit, + /// because it will handle lazy executed arguments by itself. + if (is_short_circuit_argument && !function->isShortCircuit(settings, args)) { - /// Arguments of lazy executed function can also be lazy executed. for (auto & col : columns) { if (const ColumnFunction * arg = checkAndGetShortCircuitArgument(col.column)) diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index d3e68273d03..b52a5aab256 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -138,7 +138,7 @@ public: type_indexes.insert(TypeToTypeIndex>); } - DataTypePtr getScalarType() const { return getLeastSupertype(type_indexes, true); } + DataTypePtr getScalarType() const { return getLeastSupertypeOrString(type_indexes); } bool haveNulls() const { return have_nulls; } bool needConvertField() const { return field_types.size() > 1; } @@ -167,6 +167,7 @@ FieldInfo getFieldInfo(const Field & field) ColumnObject::Subcolumn::Subcolumn(MutableColumnPtr && data_, bool is_nullable_) : least_common_type(getDataTypeByColumn(*data_)) , is_nullable(is_nullable_) + , num_rows(data_->size()) { data.push_back(std::move(data_)); } @@ -176,15 +177,13 @@ ColumnObject::Subcolumn::Subcolumn( : least_common_type(std::make_shared()) , is_nullable(is_nullable_) , num_of_defaults_in_prefix(size_) + , num_rows(size_) { } size_t ColumnObject::Subcolumn::size() const { - size_t res = num_of_defaults_in_prefix; - for (const auto & part : data) - res += part->size(); - return res; + return num_rows; } size_t ColumnObject::Subcolumn::byteSize() const @@ -321,7 +320,7 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) { if (isConversionRequiredBetweenIntegers(*base_type, *least_common_base_type)) { - base_type = getLeastSupertype(DataTypes{std::move(base_type), least_common_base_type}, true); + base_type = getLeastSupertypeOrString(DataTypes{std::move(base_type), least_common_base_type}); type_changed = true; if (!least_common_base_type->equals(*base_type)) addNewColumnPart(createArrayOfType(std::move(base_type), value_dim)); @@ -332,12 +331,14 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) field = convertFieldToTypeOrThrow(field, *least_common_type.get()); data.back()->insert(field); + ++num_rows; } void ColumnObject::Subcolumn::insertRangeFrom(const Subcolumn & src, size_t start, size_t length) { assert(start + length <= src.size()); size_t end = start + length; + num_rows += length; if (data.empty()) { @@ -345,7 +346,7 @@ void ColumnObject::Subcolumn::insertRangeFrom(const Subcolumn & src, size_t star } else if (!least_common_type.get()->equals(*src.getLeastCommonType())) { - auto new_least_common_type = getLeastSupertype(DataTypes{least_common_type.get(), src.getLeastCommonType()}, true); + auto new_least_common_type = getLeastSupertypeOrString(DataTypes{least_common_type.get(), src.getLeastCommonType()}); if (!new_least_common_type->equals(*least_common_type.get())) addNewColumnPart(std::move(new_least_common_type)); } @@ -487,6 +488,8 @@ void ColumnObject::Subcolumn::insertDefault() ++num_of_defaults_in_prefix; else data.back()->insertDefault(); + + ++num_rows; } void ColumnObject::Subcolumn::insertManyDefaults(size_t length) @@ -495,12 +498,15 @@ void ColumnObject::Subcolumn::insertManyDefaults(size_t length) num_of_defaults_in_prefix += length; else data.back()->insertManyDefaults(length); + + num_rows += length; } void ColumnObject::Subcolumn::popBack(size_t n) { assert(n <= size()); + num_rows -= n; size_t num_removed = 0; for (auto it = data.rbegin(); it != data.rend(); ++it) { @@ -559,15 +565,11 @@ ColumnObject::Subcolumn ColumnObject::Subcolumn::recreateWithDefaultValues(const if (is_nullable) scalar_type = makeNullable(scalar_type); - Subcolumn new_subcolumn; + Subcolumn new_subcolumn(*this); new_subcolumn.least_common_type = LeastCommonType{createArrayOfType(scalar_type, field_info.num_dimensions)}; - new_subcolumn.is_nullable = is_nullable; - new_subcolumn.num_of_defaults_in_prefix = num_of_defaults_in_prefix; - new_subcolumn.data.reserve(data.size()); - for (const auto & part : data) - new_subcolumn.data.push_back(recreateColumnWithDefaultValues( - part, scalar_type, field_info.num_dimensions)); + for (auto & part : new_subcolumn.data) + part = recreateColumnWithDefaultValues(part, scalar_type, field_info.num_dimensions); return new_subcolumn; } diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 89e42183ea0..4dc5bb5ce24 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -146,6 +146,8 @@ public: /// least common type and we count number of defaults in prefix, /// which will be converted to the default type of final common type. size_t num_of_defaults_in_prefix = 0; + + size_t num_rows = 0; }; using Subcolumns = SubcolumnsTree; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index f9fc14874aa..60423e2b0fe 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -489,10 +489,10 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s const T * data_pos = data.data(); /** A slightly more optimized version. - * Based on the assumption that often pieces of consecutive values - * completely pass or do not pass the filter. - * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. - */ + * Based on the assumption that often pieces of consecutive values + * completely pass or do not pass the filter. + * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. + */ static constexpr size_t SIMD_BYTES = 64; const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; @@ -577,6 +577,115 @@ ColumnPtr ColumnVector::index(const IColumn & indexes, size_t limit) const return selectIndexImpl(*this, indexes, limit); } +#ifdef __SSE2__ + +namespace +{ + /** Optimization for ColumnVector replicate using SIMD instructions. + * For such optimization it is important that data is right padded with 15 bytes. + * + * Replicate span size is offsets[i] - offsets[i - 1]. + * + * Split spans into 3 categories. + * 1. Span with 0 size. Continue iteration. + * + * 2. Span with 1 size. Update pointer from which data must be copied into result. + * Then if we see span with size 1 or greater than 1 copy data directly into result data and reset pointer. + * Example: + * Data: 1 2 3 4 + * Offsets: 1 2 3 4 + * Result data: 1 2 3 4 + * + * 3. Span with size greater than 1. Save single data element into register and copy it into result data. + * Example: + * Data: 1 2 3 4 + * Offsets: 4 4 4 4 + * Result data: 1 1 1 1 + * + * Additional handling for tail is needed if pointer from which data must be copied from span with size 1 is not null. + */ + template + requires (std::is_same_v || std::is_same_v) + void replicateSSE42Int32(const IntType * __restrict data, IntType * __restrict result_data, const IColumn::Offsets & offsets) + { + const IntType * data_copy_begin_ptr = nullptr; + size_t offsets_size = offsets.size(); + + for (size_t offset_index = 0; offset_index < offsets_size; ++offset_index) + { + size_t span = offsets[offset_index] - offsets[offset_index - 1]; + if (span == 1) + { + if (!data_copy_begin_ptr) + data_copy_begin_ptr = data + offset_index; + + continue; + } + + /// Copy data + + if (data_copy_begin_ptr) + { + size_t copy_size = (data + offset_index) - data_copy_begin_ptr; + bool remainder = copy_size % 4; + size_t sse_copy_counter = (copy_size / 4) + remainder; + auto * result_data_copy = result_data; + + while (sse_copy_counter) + { + __m128i copy_batch = _mm_loadu_si128(reinterpret_cast(data_copy_begin_ptr)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data_copy), copy_batch); + result_data_copy += 4; + data_copy_begin_ptr += 4; + --sse_copy_counter; + } + + result_data += copy_size; + data_copy_begin_ptr = nullptr; + } + + if (span == 0) + continue; + + /// Copy single data element into result data + + bool span_remainder = span % 4; + size_t copy_counter = (span / 4) + span_remainder; + auto * result_data_tmp = result_data; + __m128i copy_element_data = _mm_set1_epi32(data[offset_index]); + + while (copy_counter) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data_tmp), copy_element_data); + result_data_tmp += 4; + --copy_counter; + } + + result_data += span; + } + + /// Copy tail if needed + + if (data_copy_begin_ptr) + { + size_t copy_size = (data + offsets_size) - data_copy_begin_ptr; + bool remainder = copy_size % 4; + size_t sse_copy_counter = (copy_size / 4) + remainder; + + while (sse_copy_counter) + { + __m128i copy_batch = _mm_loadu_si128(reinterpret_cast(data_copy_begin_ptr)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data), copy_batch); + result_data += 4; + data_copy_begin_ptr += 4; + --sse_copy_counter; + } + } + } +} + +#endif + template ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const { @@ -589,6 +698,14 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); +#ifdef __SSE2__ + if constexpr (std::is_same_v) + { + replicateSSE42Int32(getData().data(), res->getData().data(), offsets); + return res; + } +#endif + auto it = res->getData().begin(); // NOLINT for (size_t i = 0; i < size; ++i) { diff --git a/src/Columns/tests/gtest_column_object.cpp b/src/Columns/tests/gtest_column_object.cpp index d5e58e5fce2..e1ad949f6a8 100644 --- a/src/Columns/tests/gtest_column_object.cpp +++ b/src/Columns/tests/gtest_column_object.cpp @@ -89,7 +89,7 @@ TEST(ColumnObject, InsertRangeFrom) const auto & type_dst = subcolumn_dst.getLeastCommonType(); const auto & type_src = subcolumn_src.getLeastCommonType(); - auto type_res = getLeastSupertype(DataTypes{type_dst, type_src}, true); + auto type_res = getLeastSupertypeOrString(DataTypes{type_dst, type_src}); size_t from = rng() % subcolumn_src.size(); size_t to = rng() % subcolumn_src.size(); diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h index 1e6e76bee91..06ccbed4064 100644 --- a/src/Common/Allocator.h +++ b/src/Common/Allocator.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #ifdef NDEBUG #define ALLOCATOR_ASLR 0 diff --git a/src/Common/Arena.h b/src/Common/Arena.h index 4c7ee458ee4..b706f3b3413 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Common/CurrentMetrics.h b/src/Common/CurrentMetrics.h index 7508c26b0ae..c184ee1e7f2 100644 --- a/src/Common/CurrentMetrics.h +++ b/src/Common/CurrentMetrics.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 6f2ac41cc08..8e7eaf4c6e6 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -631,7 +631,7 @@ M(660, HDFS_ERROR) \ M(661, CANNOT_SEND_SIGNAL) \ M(662, FS_METADATA_ERROR) \ - M(663, CANNOT_COLLECT_OBJECTS_FOR_BACKUP) \ + M(663, INCONSISTENT_METADATA_FOR_BACKUP) \ M(664, ACCESS_STORAGE_DOESNT_ALLOW_BACKUP) \ \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 253dbe6d65c..9d59fb4d7b5 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -130,7 +130,7 @@ public: int getLineNumber() const { return line_number; } void setLineNumber(int line_number_) { line_number = line_number_;} - const String getFileName() const { return file_name; } + String getFileName() const { return file_name; } void setFileName(const String & file_name_) { file_name = file_name_; } Exception * clone() const override { return new ParsingException(*this); } diff --git a/src/Common/FileCacheFactory.cpp b/src/Common/FileCacheFactory.cpp index e126ac014f2..259c1d3f48e 100644 --- a/src/Common/FileCacheFactory.cpp +++ b/src/Common/FileCacheFactory.cpp @@ -1,5 +1,4 @@ #include "FileCacheFactory.h" -#include "IFileCache.h" #include "LRUFileCache.h" namespace DB @@ -19,51 +18,66 @@ FileCacheFactory & FileCacheFactory::instance() FileCacheFactory::CacheByBasePath FileCacheFactory::getAll() { std::lock_guard lock(mutex); - return caches; + return caches_by_path; } const FileCacheSettings & FileCacheFactory::getSettings(const std::string & cache_base_path) { std::lock_guard lock(mutex); + auto it = caches_by_path.find(cache_base_path); + if (it == caches_by_path.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by path: {}", cache_base_path); + return it->second->settings; - auto * cache_data = getImpl(cache_base_path, lock); - if (cache_data) - return cache_data->settings; - - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by path: {}", cache_base_path); -} - -FileCacheFactory::CacheData * FileCacheFactory::getImpl(const std::string & cache_base_path, std::lock_guard &) -{ - auto it = caches.find(cache_base_path); - if (it == caches.end()) - return nullptr; - return &it->second; } FileCachePtr FileCacheFactory::get(const std::string & cache_base_path) { std::lock_guard lock(mutex); + auto it = caches_by_path.find(cache_base_path); + if (it == caches_by_path.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by path: {}", cache_base_path); + return it->second->cache; - auto * cache_data = getImpl(cache_base_path, lock); - if (cache_data) - return cache_data->cache; - - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by path: {}", cache_base_path); } FileCachePtr FileCacheFactory::getOrCreate( - const std::string & cache_base_path, const FileCacheSettings & file_cache_settings) + const std::string & cache_base_path, const FileCacheSettings & file_cache_settings, const std::string & name) { std::lock_guard lock(mutex); - auto * cache_data = getImpl(cache_base_path, lock); - if (cache_data) - return cache_data->cache; + auto it = caches_by_path.find(cache_base_path); + if (it != caches_by_path.end()) + { + caches_by_name.emplace(name, it->second); + return it->second->cache; + } auto cache = std::make_shared(cache_base_path, file_cache_settings); - caches.emplace(cache_base_path, CacheData(cache, file_cache_settings)); + FileCacheData result{cache, file_cache_settings}; + + auto cache_it = caches.insert(caches.end(), std::move(result)); + caches_by_name.emplace(name, cache_it); + caches_by_path.emplace(cache_base_path, cache_it); + return cache; } +FileCacheFactory::FileCacheData FileCacheFactory::getByName(const std::string & name) +{ + std::lock_guard lock(mutex); + + auto it = caches_by_name.find(name); + if (it == caches_by_name.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No cache found by name: {}", name); + + return *it->second; +} + +FileCacheFactory::CacheByName FileCacheFactory::getAllByName() +{ + std::lock_guard lock(mutex); + return caches_by_name; +} + } diff --git a/src/Common/FileCacheFactory.h b/src/Common/FileCacheFactory.h index 3518f487b6d..0d99da0343c 100644 --- a/src/Common/FileCacheFactory.h +++ b/src/Common/FileCacheFactory.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -15,20 +16,22 @@ namespace DB */ class FileCacheFactory final : private boost::noncopyable { - struct CacheData +public: + struct FileCacheData { FileCachePtr cache; FileCacheSettings settings; - CacheData(FileCachePtr cache_, const FileCacheSettings & settings_) : cache(cache_), settings(settings_) {} + FileCacheData(FileCachePtr cache_, const FileCacheSettings & settings_) : cache(cache_), settings(settings_) {} }; - using CacheByBasePath = std::unordered_map; + using Caches = std::list; + using CacheByBasePath = std::unordered_map; + using CacheByName = std::unordered_map; -public: static FileCacheFactory & instance(); - FileCachePtr getOrCreate(const std::string & cache_base_path, const FileCacheSettings & file_cache_settings); + FileCachePtr getOrCreate(const std::string & cache_base_path, const FileCacheSettings & file_cache_settings, const std::string & name); FileCachePtr get(const std::string & cache_base_path); @@ -36,11 +39,16 @@ public: const FileCacheSettings & getSettings(const std::string & cache_base_path); -private: - CacheData * getImpl(const std::string & cache_base_path, std::lock_guard &); + FileCacheData getByName(const std::string & name); + CacheByName getAllByName(); + +private: std::mutex mutex; - CacheByBasePath caches; + Caches caches; + + CacheByBasePath caches_by_path; + CacheByName caches_by_name; }; } diff --git a/src/Common/FileCacheSettings.h b/src/Common/FileCacheSettings.h index 1d8b613bedd..c04cc915486 100644 --- a/src/Common/FileCacheSettings.h +++ b/src/Common/FileCacheSettings.h @@ -2,7 +2,7 @@ #include -namespace Poco { namespace Util { class AbstractConfiguration; } } +namespace Poco { namespace Util { class AbstractConfiguration; } } // NOLINT(cppcoreguidelines-virtual-class-destructor) namespace DB { diff --git a/src/Common/IFactoryWithAliases.h b/src/Common/IFactoryWithAliases.h index 458adc4522f..8feac867dd2 100644 --- a/src/Common/IFactoryWithAliases.h +++ b/src/Common/IFactoryWithAliases.h @@ -27,9 +27,9 @@ protected: String getAliasToOrName(const String & name) const { - if (aliases.count(name)) + if (aliases.contains(name)) return aliases.at(name); - else if (String name_lowercase = Poco::toLower(name); case_insensitive_aliases.count(name_lowercase)) + else if (String name_lowercase = Poco::toLower(name); case_insensitive_aliases.contains(name_lowercase)) return case_insensitive_aliases.at(name_lowercase); else return name; @@ -108,7 +108,7 @@ public: bool isAlias(const String & name) const { - return aliases.count(name) || case_insensitive_aliases.count(name); + return aliases.count(name) || case_insensitive_aliases.contains(name); } bool hasNameOrAlias(const String & name) const @@ -125,7 +125,7 @@ public: return name; } - virtual ~IFactoryWithAliases() override = default; + ~IFactoryWithAliases() override = default; private: using InnerMap = std::unordered_map; // name -> creator diff --git a/src/Common/IntervalKind.cpp b/src/Common/IntervalKind.cpp index 75c2a83e9fb..d3cd4eeff8f 100644 --- a/src/Common/IntervalKind.cpp +++ b/src/Common/IntervalKind.cpp @@ -9,13 +9,13 @@ namespace ErrorCodes extern const int SYNTAX_ERROR; } -Float64 IntervalKind::toAvgSeconds() const +Int32 IntervalKind::toAvgSeconds() const { switch (kind) { - case IntervalKind::Nanosecond: return 0.000000001; - case IntervalKind::Microsecond: return 0.000001; - case IntervalKind::Millisecond: return 0.001; + case IntervalKind::Nanosecond: + case IntervalKind::Microsecond: + case IntervalKind::Millisecond: return 0; /// fractional parts of seconds have 0 seconds case IntervalKind::Second: return 1; case IntervalKind::Minute: return 60; case IntervalKind::Hour: return 3600; @@ -28,25 +28,6 @@ Float64 IntervalKind::toAvgSeconds() const __builtin_unreachable(); } -bool IntervalKind::isFixedLength() const -{ - switch (kind) - { - case IntervalKind::Nanosecond: - case IntervalKind::Microsecond: - case IntervalKind::Millisecond: - case IntervalKind::Second: - case IntervalKind::Minute: - case IntervalKind::Hour: - case IntervalKind::Day: - case IntervalKind::Week: return true; - case IntervalKind::Month: - case IntervalKind::Quarter: - case IntervalKind::Year: return false; - } - __builtin_unreachable(); -} - IntervalKind IntervalKind::fromAvgSeconds(Int64 num_seconds) { if (num_seconds) diff --git a/src/Common/IntervalKind.h b/src/Common/IntervalKind.h index 65c14515e34..d5f2b5672cd 100644 --- a/src/Common/IntervalKind.h +++ b/src/Common/IntervalKind.h @@ -31,15 +31,12 @@ struct IntervalKind /// Returns number of seconds in one interval. /// For `Month`, `Quarter` and `Year` the function returns an average number of seconds. - Float64 toAvgSeconds() const; + Int32 toAvgSeconds() const; /// Chooses an interval kind based on number of seconds. /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Hour`. static IntervalKind fromAvgSeconds(Int64 num_seconds); - /// Returns whether IntervalKind has a fixed number of seconds (e.g. Day) or non-fixed(e.g. Month) - bool isFixedLength() const; - /// Returns an uppercased version of what `toString()` returns. const char * toKeyword() const; diff --git a/src/Common/LocalDate.h b/src/Common/LocalDate.h index 5d070e77647..dc36f92bebf 100644 --- a/src/Common/LocalDate.h +++ b/src/Common/LocalDate.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 51f4c83dc23..a9aef7d8465 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -338,11 +338,8 @@ void MemoryTracker::free(Int64 size) accounted_size += new_amount; } } - if (!OvercommitTrackerBlockerInThread::isBlocked()) - { - if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed); overcommit_tracker_ptr) - overcommit_tracker_ptr->tryContinueQueryExecutionAfterFree(accounted_size); - } + if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed)) + overcommit_tracker_ptr->tryContinueQueryExecutionAfterFree(accounted_size); if (auto * loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 3cef72eb8b4..0dea6589adc 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -20,11 +20,32 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , global_mutex(global_mutex_) , freed_memory(0) , required_memory(0) + , next_id(0) + , id_to_release(0) , allow_release(true) {} +#define LOG_DEBUG_SAFE(...) \ + do { \ + OvercommitTrackerBlockerInThread blocker; \ + try \ + { \ + ALLOW_ALLOCATIONS_IN_SCOPE; \ + LOG_DEBUG(__VA_ARGS__); \ + } \ + catch (...) \ + { \ + if (fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n") != -1) \ + ; \ + } \ + } while (false) + OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { + DENY_ALLOCATIONS_IN_SCOPE; + + if (OvercommitTrackerBlockerInThread::isBlocked()) + return OvercommitResult::NONE; // NOTE: Do not change the order of locks // // global_mutex must be acquired before overcommit_m, because @@ -34,6 +55,8 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int std::unique_lock global_lock(global_mutex); std::unique_lock lk(overcommit_m); + size_t id = next_id++; + auto max_wait_time = tracker->getOvercommitWaitingTime(); if (max_wait_time == ZERO_MICROSEC) @@ -65,23 +88,21 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int allow_release = true; required_memory += amount; - required_per_thread[tracker] = amount; auto wait_start_time = std::chrono::system_clock::now(); - bool timeout = !cv.wait_for(lk, max_wait_time, [this, tracker]() + bool timeout = !cv.wait_for(lk, max_wait_time, [this, id]() { - return required_per_thread[tracker] == 0 || cancellation_state == QueryCancellationState::NONE; + return id < id_to_release || cancellation_state == QueryCancellationState::NONE; }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); - LOG_DEBUG(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); + LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); required_memory -= amount; - Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 - required_per_thread.erase(tracker); + bool still_need = !(id < id_to_release); // True if thread wasn't released // If threads where not released since last call of this method, // we can release them now. - if (allow_release && required_memory <= freed_memory && still_need != 0) + if (allow_release && required_memory <= freed_memory && still_need) releaseThreads(); // All required amount of memory is free now and selected query to stop doesn't know about it. @@ -90,7 +111,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int reset(); if (timeout) return OvercommitResult::TIMEOUTED; - if (still_need != 0) + if (still_need) return OvercommitResult::NOT_ENOUGH_FREED; else return OvercommitResult::MEMORY_FREED; @@ -98,6 +119,11 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) { + DENY_ALLOCATIONS_IN_SCOPE; + + if (OvercommitTrackerBlockerInThread::isBlocked()) + return; + std::lock_guard guard(overcommit_m); if (cancellation_state != QueryCancellationState::NONE) { @@ -109,10 +135,12 @@ void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) void OvercommitTracker::onQueryStop(MemoryTracker * tracker) { - std::unique_lock lk(overcommit_m); + DENY_ALLOCATIONS_IN_SCOPE; + + std::lock_guard lk(overcommit_m); if (picked_tracker == tracker) { - LOG_DEBUG(getLogger(), "Picked query stopped"); + LOG_DEBUG_SAFE(getLogger(), "Picked query stopped"); reset(); cv.notify_all(); @@ -121,8 +149,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) void OvercommitTracker::releaseThreads() { - for (auto & required : required_per_thread) - required.second = 0; + id_to_release = next_id; freed_memory = 0; allow_release = false; // To avoid repeating call of this method in OvercommitTracker::needToStopQuery cv.notify_all(); @@ -140,7 +167,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. auto & queries = user_process_list->queries; - LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", queries.size()); + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()); for (auto const & query : queries) { if (query.second->isKilled()) @@ -151,14 +178,14 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() continue; auto ratio = memory_tracker->getOvercommitRatio(); - LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); if (ratio.soft_limit != 0 && current_ratio < ratio) { query_tracker = memory_tracker; current_ratio = ratio; } } - LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", + LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", current_ratio.committed, current_ratio.soft_limit); picked_tracker = query_tracker; } @@ -174,7 +201,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() OvercommitRatio current_ratio{0, 0}; // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. - LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", process_list->size()); + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()); for (auto const & query : process_list->processes) { if (query.isKilled()) @@ -190,14 +217,14 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() if (!memory_tracker) continue; auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit); - LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); if (current_ratio < ratio) { query_tracker = memory_tracker; current_ratio = ratio; } } - LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", + LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", current_ratio.committed, current_ratio.soft_limit); picked_tracker = query_tracker; } diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 80aaed68e37..6f6788493a9 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -104,6 +104,10 @@ private: picked_tracker = nullptr; cancellation_state = QueryCancellationState::NONE; freed_memory = 0; + + next_id = 0; + id_to_release = 0; + allow_release = true; } @@ -111,8 +115,6 @@ private: QueryCancellationState cancellation_state; - std::unordered_map required_per_thread; - // Global mutex which is used in ProcessList to synchronize // insertion and deletion of queries. // OvercommitTracker::pickQueryToExcludeImpl() implementations @@ -122,6 +124,9 @@ private: Int64 freed_memory; Int64 required_memory; + size_t next_id; // Id provided to the next thread to come in OvercommitTracker + size_t id_to_release; // We can release all threads with id smaller than this + bool allow_release; }; diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index e4c1969383b..44be28eab75 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Common/PoolBase.h b/src/Common/PoolBase.h index 4b4e2c5cfa7..dac126fb5d7 100644 --- a/src/Common/PoolBase.h +++ b/src/Common/PoolBase.h @@ -55,7 +55,7 @@ private: explicit PoolEntryHelper(PooledObject & data_) : data(data_) { data.in_use = true; } ~PoolEntryHelper() { - std::unique_lock lock(data.pool.mutex); + std::lock_guard lock(data.pool.mutex); data.in_use = false; data.pool.available.notify_one(); } @@ -163,7 +163,7 @@ public: inline size_t size() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); return items.size(); } diff --git a/src/Common/ProfileEvents.h b/src/Common/ProfileEvents.h index 14d134a14a0..6eebb75c5ca 100644 --- a/src/Common/ProfileEvents.h +++ b/src/Common/ProfileEvents.h @@ -4,7 +4,7 @@ #include "base/types.h" #include #include -#include +#include /** Implements global counters for various events happening in the application * - for high level profiling. diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 33c408e6d8a..84a2e9d1f7f 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #ifdef OS_DARWIN // ucontext is not available without _XOPEN_SOURCE diff --git a/src/Common/StringSearcher.h b/src/Common/StringSearcher.h index b556ace75a7..b8f8a9d3a88 100644 --- a/src/Common/StringSearcher.h +++ b/src/Common/StringSearcher.h @@ -7,8 +7,8 @@ #include #include #include -#include -#include +#include +#include #ifdef __SSE2__ #include @@ -826,66 +826,44 @@ using UTF8CaseInsensitiveStringSearcher = StringSearcher; using ASCIICaseSensitiveTokenSearcher = TokenSearcher; using ASCIICaseInsensitiveTokenSearcher = TokenSearcher; - -/** Uses functions from libc. - * It makes sense to use only with short haystacks when cheap initialization is required. - * There is no option for case-insensitive search for UTF-8 strings. - * It is required that strings are zero-terminated. - */ - -struct LibCASCIICaseSensitiveStringSearcher : public StringSearcherBase +/// Use only with short haystacks where cheap initialization is required. +template +struct StdLibASCIIStringSearcher : public StringSearcherBase { - const char * const needle; + const char * const needle_start; + const char * const needle_end; template requires (sizeof(CharT) == 1) - LibCASCIICaseSensitiveStringSearcher(const CharT * const needle_, const size_t /* needle_size */) - : needle(reinterpret_cast(needle_)) {} + StdLibASCIIStringSearcher(const CharT * const needle_start_, const size_t needle_size_) + : needle_start{reinterpret_cast(needle_start_)} + , needle_end{reinterpret_cast(needle_start) + needle_size_} + {} template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const CharT * const haystack_end) const + const CharT * search(const CharT * haystack_start, const CharT * const haystack_end) const { - const auto * res = strstr(reinterpret_cast(haystack), reinterpret_cast(needle)); - if (!res) - return haystack_end; - return reinterpret_cast(res); + if constexpr (CaseInsensitive) + { + return std::search( + haystack_start, haystack_end, needle_start, needle_end, + [](char c1, char c2) {return std::toupper(c1) == std::toupper(c2);}); + } + else + { + return std::search( + haystack_start, haystack_end, needle_start, needle_end, + [](char c1, char c2) {return c1 == c2;}); + } } template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const + const CharT * search(const CharT * haystack_start, const size_t haystack_length) const { - return search(haystack, haystack + haystack_size); + return search(haystack_start, haystack_start + haystack_length); } }; -struct LibCASCIICaseInsensitiveStringSearcher : public StringSearcherBase -{ - const char * const needle; - - template - requires (sizeof(CharT) == 1) - LibCASCIICaseInsensitiveStringSearcher(const CharT * const needle_, const size_t /* needle_size */) - : needle(reinterpret_cast(needle_)) {} - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const CharT * const haystack_end) const - { - const auto * res = strcasestr(reinterpret_cast(haystack), reinterpret_cast(needle)); - if (!res) - return haystack_end; - return reinterpret_cast(res); - } - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const - { - return search(haystack, haystack + haystack_size); - } -}; - - } diff --git a/src/Common/SystemLogBase.cpp b/src/Common/SystemLogBase.cpp index 7128d00714c..67aedbd5670 100644 --- a/src/Common/SystemLogBase.cpp +++ b/src/Common/SystemLogBase.cpp @@ -139,7 +139,7 @@ void SystemLogBase::flush(bool force) uint64_t this_thread_requested_offset; { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (is_shutdown) return; diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index a76037ae5cf..3f5091af0c9 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -209,7 +209,7 @@ template void ThreadPoolImpl::finalize() { { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); shutdown = true; } @@ -224,14 +224,14 @@ void ThreadPoolImpl::finalize() template size_t ThreadPoolImpl::active() const { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); return scheduled_jobs; } template bool ThreadPoolImpl::finished() const { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); return shutdown; } @@ -290,7 +290,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ job = {}; { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!first_exception) first_exception = std::current_exception(); // NOLINT if (shutdown_on_exception) @@ -305,7 +305,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); --scheduled_jobs; if (threads.size() > scheduled_jobs + max_free_threads) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index b5c2e7be11f..a4f99c1be1a 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -148,28 +148,22 @@ ThreadStatus::ThreadStatus() ThreadStatus::~ThreadStatus() { - try - { - if (untracked_memory > 0) - memory_tracker.alloc(untracked_memory); - else - memory_tracker.free(-untracked_memory); - } - catch (const DB::Exception &) - { - /// It's a minor tracked memory leak here (not the memory itself but it's counter). - /// We've already allocated a little bit more than the limit and cannot track it in the thread memory tracker or its parent. - tryLogCurrentException(log); - } + if (untracked_memory > 0) + memory_tracker.allocNoThrow(untracked_memory); + else + memory_tracker.free(-untracked_memory); if (thread_group) { - std::lock_guard guard(thread_group->mutex); - thread_group->finished_threads_counters_memory.emplace_back(ThreadGroupStatus::ProfileEventsCountersAndMemory{ + ThreadGroupStatus::ProfileEventsCountersAndMemory counters + { performance_counters.getPartiallyAtomicSnapshot(), memory_tracker.get(), - thread_id, - }); + thread_id + }; + + std::lock_guard guard(thread_group->mutex); + thread_group->finished_threads_counters_memory.emplace_back(std::move(counters)); thread_group->threads.erase(this); } diff --git a/src/Common/Volnitsky.h b/src/Common/Volnitsky.h index 7eca0c0fe53..a6aef293ac1 100644 --- a/src/Common/Volnitsky.h +++ b/src/Common/Volnitsky.h @@ -476,7 +476,7 @@ class MultiVolnitskyBase { private: /// needles and their offsets - const std::vector & needles; + const std::vector & needles; /// fallback searchers @@ -502,7 +502,7 @@ private: static constexpr size_t small_limit = VolnitskyTraits::hash_size / 8; public: - explicit MultiVolnitskyBase(const std::vector & needles_) : needles{needles_}, step{0}, last{0} + explicit MultiVolnitskyBase(const std::vector & needles_) : needles{needles_}, step{0}, last{0} { fallback_searchers.reserve(needles.size()); hash = std::unique_ptr(new OffsetId[VolnitskyTraits::hash_size]); /// No zero initialization, it will be done later. @@ -535,8 +535,8 @@ public: for (; last < size; ++last) { - const char * cur_needle_data = needles[last].data; - const size_t cur_needle_size = needles[last].size; + const char * cur_needle_data = needles[last].data(); + const size_t cur_needle_size = needles[last].size(); /// save the indices of fallback searchers if (VolnitskyTraits::isFallbackNeedle(cur_needle_size)) @@ -593,7 +593,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) + if (res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) return true; } } @@ -625,7 +625,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) + if (res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) answer = std::min(answer, ind); } } @@ -663,7 +663,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) + if (res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) answer = std::min(answer, res - haystack); } } @@ -699,7 +699,7 @@ public: const auto * res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; if (answer[ind] == 0 - && res + needles[ind].size <= haystack_end + && res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) answer[ind] = count_chars(haystack, res); } diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 3b05e671384..b917a0a1d13 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -1,12 +1,10 @@ #include "filesystemHelpers.h" #if defined(OS_LINUX) -# include # include # include #endif #include -#include #include #include #include diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h index d6efeed17e6..83b9d6e9fb1 100644 --- a/src/Common/formatIPv6.h +++ b/src/Common/formatIPv6.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/src/Common/memcpySmall.h b/src/Common/memcpySmall.h index 6cadc19262f..4f38095c7f1 100644 --- a/src/Common/memcpySmall.h +++ b/src/Common/memcpySmall.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #ifdef __SSE2__ # include diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index d4c188fe8d9..8a46d8ee296 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -576,7 +576,7 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ auto set_initialized = [this]() { - std::unique_lock lock(initialized_mutex); + std::lock_guard lock(initialized_mutex); initialized_flag = true; initialized_cv.notify_all(); }; diff --git a/src/Core/Field.h b/src/Core/Field.h index 2a925a9e2a6..4948ec4ae61 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -54,7 +54,7 @@ DEFINE_FIELD_VECTOR(Map); /// TODO: use map instead of vector. #undef DEFINE_FIELD_VECTOR -using FieldMap = std::map, AllocatorWithMemoryTracking>>; +using FieldMap = std::map, AllocatorWithMemoryTracking>>; #define DEFINE_FIELD_MAP(X) \ struct X : public FieldMap \ diff --git a/src/Core/PostgreSQL/Connection.h b/src/Core/PostgreSQL/Connection.h index 8c5609dc66b..97ce3c152d5 100644 --- a/src/Core/PostgreSQL/Connection.h +++ b/src/Core/PostgreSQL/Connection.h @@ -32,7 +32,7 @@ struct ConnectionInfo class Connection : private boost::noncopyable { public: - Connection(const ConnectionInfo & connection_info_, bool replication_ = false, size_t num_tries = 3); + explicit Connection(const ConnectionInfo & connection_info_, bool replication_ = false, size_t num_tries = 3); void execWithRetry(const std::function & exec); diff --git a/src/Core/PostgreSQL/PoolWithFailover.h b/src/Core/PostgreSQL/PoolWithFailover.h index 600e12fb53a..4e3a17b5e9c 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.h +++ b/src/Core/PostgreSQL/PoolWithFailover.h @@ -25,13 +25,13 @@ public: static constexpr inline auto POSTGRESQL_POOL_WAIT_TIMEOUT = 5000; static constexpr inline auto POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES = 5; - PoolWithFailover( + explicit PoolWithFailover( const DB::ExternalDataSourcesConfigurationByPriority & configurations_by_priority, size_t pool_size = POSTGRESQL_POOL_DEFAULT_SIZE, size_t pool_wait_timeout = POSTGRESQL_POOL_WAIT_TIMEOUT, size_t max_tries_ = POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); - PoolWithFailover( + explicit PoolWithFailover( const DB::StoragePostgreSQLConfiguration & configuration, size_t pool_size = POSTGRESQL_POOL_DEFAULT_SIZE, size_t pool_wait_timeout = POSTGRESQL_POOL_WAIT_TIMEOUT, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f1fd9d20f00..f1f7bc7f2ab 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -445,7 +445,6 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Seconds, wait_for_window_view_fire_signal_timeout, 10, "Timeout for waiting for window view fire signal in event time processing", 0) \ M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ - M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic, "Default database engine.", 0) \ M(DefaultTableEngine, default_table_engine, DefaultTableEngine::None, "Default table engine used when ENGINE is not set in CREATE statement.",0) \ M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \ @@ -591,6 +590,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, use_local_cache_for_remote_storage, true, "Use local cache for remote storage like HDFS or S3, it's used for remote table engine only", 0) \ \ M(Bool, allow_unrestricted_reads_from_keeper, false, "Allow unrestricted (without condition on path) reads from system.zookeeper table, can be handy, but is not safe for zookeeper", 0) \ + M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ + M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ @@ -601,6 +602,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \ M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \ + M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ @@ -637,6 +639,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) MAKE_OBSOLETE(M, UInt64, background_schedule_pool_size, 128) \ MAKE_OBSOLETE(M, UInt64, background_message_broker_schedule_pool_size, 16) \ MAKE_OBSOLETE(M, UInt64, background_distributed_schedule_pool_size, 16) \ + MAKE_OBSOLETE(M, DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic) \ /** The section above is for obsolete settings. Do not add anything there. */ diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index e731787a5c1..445aa4463bd 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -925,7 +925,7 @@ void BaseDaemon::handleSignal(int signal_id) signal_id == SIGQUIT || signal_id == SIGTERM) { - std::unique_lock lock(signal_handler_mutex); + std::lock_guard lock(signal_handler_mutex); { ++terminate_signals_counter; sigint_signals_counter += signal_id == SIGINT; diff --git a/src/DataTypes/FieldToDataType.cpp b/src/DataTypes/FieldToDataType.cpp index 283d1b1e41a..49ece27cc1f 100644 --- a/src/DataTypes/FieldToDataType.cpp +++ b/src/DataTypes/FieldToDataType.cpp @@ -22,13 +22,14 @@ namespace ErrorCodes extern const int EMPTY_DATA_PASSED; } - -DataTypePtr FieldToDataType::operator() (const Null &) const +template +DataTypePtr FieldToDataType::operator() (const Null &) const { return std::make_shared(std::make_shared()); } -DataTypePtr FieldToDataType::operator() (const UInt64 & x) const +template +DataTypePtr FieldToDataType::operator() (const UInt64 & x) const { if (x <= std::numeric_limits::max()) return std::make_shared(); if (x <= std::numeric_limits::max()) return std::make_shared(); @@ -36,7 +37,8 @@ DataTypePtr FieldToDataType::operator() (const UInt64 & x) const return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Int64 & x) const +template +DataTypePtr FieldToDataType::operator() (const Int64 & x) const { if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); @@ -44,77 +46,90 @@ DataTypePtr FieldToDataType::operator() (const Int64 & x) const return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Float64 &) const +template +DataTypePtr FieldToDataType::operator() (const Float64 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const UInt128 &) const +template +DataTypePtr FieldToDataType::operator() (const UInt128 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Int128 &) const +template +DataTypePtr FieldToDataType::operator() (const Int128 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const UInt256 &) const +template +DataTypePtr FieldToDataType::operator() (const UInt256 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Int256 &) const +template +DataTypePtr FieldToDataType::operator() (const Int256 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const UUID &) const +template +DataTypePtr FieldToDataType::operator() (const UUID &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const String &) const +template +DataTypePtr FieldToDataType::operator() (const String &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const Array & x) const +template +DataTypePtr FieldToDataType::operator() (const Array & x) const { DataTypes element_types; element_types.reserve(x.size()); for (const Field & elem : x) - element_types.emplace_back(applyVisitor(FieldToDataType(allow_convertion_to_string), elem)); + element_types.emplace_back(applyVisitor(*this, elem)); - return std::make_shared(getLeastSupertype(element_types, allow_convertion_to_string)); + return std::make_shared(getLeastSupertype(element_types)); } -DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const +template +DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const { if (tuple.empty()) throw Exception("Cannot infer type of an empty tuple", ErrorCodes::EMPTY_DATA_PASSED); @@ -123,12 +138,13 @@ DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const element_types.reserve(tuple.size()); for (const auto & element : tuple) - element_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), element)); + element_types.push_back(applyVisitor(*this, element)); return std::make_shared(element_types); } -DataTypePtr FieldToDataType::operator() (const Map & map) const +template +DataTypePtr FieldToDataType::operator() (const Map & map) const { DataTypes key_types; DataTypes value_types; @@ -139,30 +155,37 @@ DataTypePtr FieldToDataType::operator() (const Map & map) const { const auto & tuple = elem.safeGet(); assert(tuple.size() == 2); - key_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), tuple[0])); - value_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), tuple[1])); + key_types.push_back(applyVisitor(*this, tuple[0])); + value_types.push_back(applyVisitor(*this, tuple[1])); } return std::make_shared( - getLeastSupertype(key_types, allow_convertion_to_string), - getLeastSupertype(value_types, allow_convertion_to_string)); + getLeastSupertype(key_types), + getLeastSupertype(value_types)); } -DataTypePtr FieldToDataType::operator() (const Object &) const +template +DataTypePtr FieldToDataType::operator() (const Object &) const { /// TODO: Do we need different parameters for type Object? return std::make_shared("json", false); } -DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const +template +DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const { const auto & name = static_cast(x).name; return DataTypeFactory::instance().get(name); } -DataTypePtr FieldToDataType::operator()(const bool &) const +template +DataTypePtr FieldToDataType::operator()(const bool &) const { return DataTypeFactory::instance().get("Bool"); } +template class FieldToDataType; +template class FieldToDataType; +template class FieldToDataType; + } diff --git a/src/DataTypes/FieldToDataType.h b/src/DataTypes/FieldToDataType.h index 1922ac8b746..5e66fe420ad 100644 --- a/src/DataTypes/FieldToDataType.h +++ b/src/DataTypes/FieldToDataType.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -17,14 +18,10 @@ using DataTypePtr = std::shared_ptr; * Note that you still have to convert Field to corresponding data type before inserting to columns * (for example, this is necessary to convert elements of Array to common type). */ +template class FieldToDataType : public StaticVisitor { public: - FieldToDataType(bool allow_convertion_to_string_ = false) - : allow_convertion_to_string(allow_convertion_to_string_) - { - } - DataTypePtr operator() (const Null & x) const; DataTypePtr operator() (const UInt64 & x) const; DataTypePtr operator() (const UInt128 & x) const; @@ -45,9 +42,6 @@ public: DataTypePtr operator() (const UInt256 & x) const; DataTypePtr operator() (const Int256 & x) const; DataTypePtr operator() (const bool & x) const; - -private: - bool allow_convertion_to_string; }; } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 420ef61a13f..fce8906abe5 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -565,4 +565,31 @@ class DataTypeEnum; template inline constexpr bool IsDataTypeEnum> = true; +#define FOR_BASIC_NUMERIC_TYPES(M) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(Int8) \ + M(Int16) \ + M(Int32) \ + M(Int64) \ + M(Float32) \ + M(Float64) + +#define FOR_NUMERIC_TYPES(M) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(UInt128) \ + M(UInt256) \ + M(Int8) \ + M(Int16) \ + M(Int32) \ + M(Int64) \ + M(Int128) \ + M(Int256) \ + M(Float32) \ + M(Float64) } diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 20aae08f363..5dae2b7b413 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -1,8 +1,10 @@ #include +#include #include #include #include +#include "Columns/IColumn.h" #include #include @@ -17,7 +19,6 @@ #include - namespace DB { @@ -76,8 +77,7 @@ Block flatten(const Block & block) for (const auto & elem : block) { - const DataTypeArray * type_arr = typeid_cast(elem.type.get()); - if (type_arr) + if (const DataTypeArray * type_arr = typeid_cast(elem.type.get())) { const DataTypeTuple * type_tuple = typeid_cast(type_arr->getNestedType().get()); if (type_tuple && type_tuple->haveExplicitNames()) @@ -106,7 +106,7 @@ Block flatten(const Block & block) res.insert(ColumnWithTypeAndName( is_const ? ColumnConst::create(std::move(column_array_of_element), block.rows()) - : std::move(column_array_of_element), + : column_array_of_element, std::make_shared(element_types[i]), nested_name)); } @@ -114,6 +114,28 @@ Block flatten(const Block & block) else res.insert(elem); } + else if (const DataTypeTuple * type_tuple = typeid_cast(elem.type.get())) + { + if (type_tuple->haveExplicitNames()) + { + const DataTypes & element_types = type_tuple->getElements(); + const Strings & names = type_tuple->getElementNames(); + const ColumnTuple * column_tuple; + if (isColumnConst(*elem.column)) + column_tuple = typeid_cast(&assert_cast(*elem.column).getDataColumn()); + else + column_tuple = typeid_cast(elem.column.get()); + size_t tuple_size = column_tuple->tupleSize(); + for (size_t i = 0; i < tuple_size; ++i) + { + const auto & element_column = column_tuple->getColumn(i); + String nested_name = concatenateName(elem.name, names[i]); + res.insert(ColumnWithTypeAndName(element_column.getPtr(), element_types[i], nested_name)); + } + } + else + res.insert(elem); + } else res.insert(elem); } @@ -243,7 +265,71 @@ std::unordered_set getAllTableNames(const Block & block, bool to_lower_c } return nested_table_names; } - } +NestedColumnExtractHelper::NestedColumnExtractHelper(const Block & block_, bool case_insentive_) + : block(block_) + , case_insentive(case_insentive_) +{} + +std::optional NestedColumnExtractHelper::extractColumn(const String & column_name) +{ + if (block.has(column_name, case_insentive)) + return {block.getByName(column_name, case_insentive)}; + + auto nested_names = Nested::splitName(column_name); + if (case_insentive) + { + boost::to_lower(nested_names.first); + boost::to_lower(nested_names.second); + } + if (!block.has(nested_names.first, case_insentive)) + return {}; + + if (!nested_tables.contains(nested_names.first)) + { + ColumnsWithTypeAndName columns = {block.getByName(nested_names.first, case_insentive)}; + nested_tables[nested_names.first] = std::make_shared(Nested::flatten(columns)); + } + + return extractColumn(column_name, nested_names.first, nested_names.second); +} + +std::optional NestedColumnExtractHelper::extractColumn( + const String & original_column_name, const String & column_name_prefix, const String & column_name_suffix) +{ + auto table_iter = nested_tables.find(column_name_prefix); + if (table_iter == nested_tables.end()) + { + return {}; + } + + auto & nested_table = table_iter->second; + auto nested_names = Nested::splitName(column_name_suffix); + auto new_column_name_prefix = Nested::concatenateName(column_name_prefix, nested_names.first); + if (nested_names.second.empty()) + { + if (auto * column_ref = nested_table->findByName(new_column_name_prefix, case_insentive)) + { + ColumnWithTypeAndName column = *column_ref; + if (case_insentive) + column.name = original_column_name; + return {std::move(column)}; + } + else + { + return {}; + } + } + + if (!nested_table->has(new_column_name_prefix, case_insentive)) + { + return {}; + } + + ColumnsWithTypeAndName columns = {nested_table->getByName(new_column_name_prefix, case_insentive)}; + Block sub_block(columns); + nested_tables[new_column_name_prefix] = std::make_shared(Nested::flatten(sub_block)); + return extractColumn(original_column_name, new_column_name_prefix, nested_names.second); +} } diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index f6dc42d5c58..e7cda541f47 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -18,8 +18,9 @@ namespace Nested /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot. std::string extractTableName(const std::string & nested_name); - /// Replace Array(Tuple(...)) columns to a multiple of Array columns in a form of `column_name.element_name`. - /// only for named tuples that actually represent Nested structures. + /// Flat a column of nested type into columns + /// 1) For named tuples，t Tuple(x .., y ..., ...), replace it with t.x ..., t.y ... , ... + /// 2) For an Array with named Tuple element column, a Array(Tuple(x ..., y ..., ...)), replace it with multiple Array Columns, a.x ..., a.y ..., ... Block flatten(const Block & block); /// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column. @@ -35,4 +36,20 @@ namespace Nested std::unordered_set getAllTableNames(const Block & block, bool to_lower_case = false); } +/// Use this class to extract element columns from columns of nested type in a block, e.g. named Tuple. +/// It can extract a column from a multiple nested type column, e.g. named Tuple in named Tuple +/// Keeps some intermediate data to avoid rebuild them multi-times. +class NestedColumnExtractHelper +{ +public: + explicit NestedColumnExtractHelper(const Block & block_, bool case_insentive_); + std::optional extractColumn(const String & column_name); +private: + std::optional + extractColumn(const String & original_column_name, const String & column_name_prefix, const String & column_name_suffix); + const Block & block; + bool case_insentive; + std::map nested_tables; +}; + } diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index df639ae7aab..b81c8c7a033 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -261,7 +261,7 @@ DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambi key.getPath(), subtypes[0]->getName(), subtypes[i]->getName()); tuple_paths.emplace_back(key); - tuple_types.emplace_back(getLeastSupertype(subtypes, /*allow_conversion_to_string=*/ true)); + tuple_types.emplace_back(getLeastSupertypeOrString(subtypes)); } if (tuple_paths.empty()) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 85831df271a..cf49fa8798d 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -61,29 +61,23 @@ void SerializationObject::deserializeTextImpl(IColumn & column, Reader & auto & [paths, values] = *result; assert(paths.size() == values.size()); - HashSet paths_set; - size_t column_size = column_object.size(); - + size_t old_column_size = column_object.size(); for (size_t i = 0; i < paths.size(); ++i) { auto field_info = getFieldInfo(values[i]); if (isNothing(field_info.scalar_type)) continue; - if (!paths_set.insert(paths[i].getPath()).second) - throw Exception(ErrorCodes::INCORRECT_DATA, - "Object has ambiguous path: {}", paths[i].getPath()); - if (!column_object.hasSubcolumn(paths[i])) { if (paths[i].hasNested()) - column_object.addNestedSubcolumn(paths[i], field_info, column_size); + column_object.addNestedSubcolumn(paths[i], field_info, old_column_size); else - column_object.addSubcolumn(paths[i], column_size); + column_object.addSubcolumn(paths[i], old_column_size); } auto & subcolumn = column_object.getSubcolumn(paths[i]); - assert(subcolumn.size() == column_size); + assert(subcolumn.size() == old_column_size); subcolumn.insert(std::move(values[i]), std::move(field_info)); } @@ -92,7 +86,7 @@ void SerializationObject::deserializeTextImpl(IColumn & column, Reader & const auto & subcolumns = column_object.getSubcolumns(); for (const auto & entry : subcolumns) { - if (!paths_set.has(entry->path.getPath())) + if (entry->data.size() == old_column_size) { bool inserted = column_object.tryInsertDefaultFromNested(entry); if (!inserted) diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index 14cfafb1ef3..ac240c4afce 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -55,16 +55,24 @@ String getExceptionMessagePrefix(const DataTypes & types) return res.str(); } -DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_string) +template +DataTypePtr throwOrReturn(const DataTypes & types, std::string_view message_suffix, int error_code) { - auto throw_or_return = [&](std::string_view message, int error_code) - { - if (allow_conversion_to_string) - return std::make_shared(); + if constexpr (on_error == LeastSupertypeOnError::String) + return std::make_shared(); - throw Exception(String(message), error_code); - }; + if constexpr (on_error == LeastSupertypeOnError::Null) + return nullptr; + if (message_suffix.empty()) + throw Exception(error_code, getExceptionMessagePrefix(types)); + + throw Exception(error_code, "{} {}", getExceptionMessagePrefix(types), message_suffix); +} + +template +DataTypePtr getNumericType(const TypeIndexSet & types) +{ bool all_numbers = true; size_t max_bits_of_signed_integer = 0; @@ -107,14 +115,14 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ maximize(max_mantissa_bits_of_floating, 24); else if (type == TypeIndex::Float64) maximize(max_mantissa_bits_of_floating, 53); - else + else if (type != TypeIndex::Nothing) all_numbers = false; } if (max_bits_of_signed_integer || max_bits_of_unsigned_integer || max_mantissa_bits_of_floating) { if (!all_numbers) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are numbers and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are numbers and some of them are not", ErrorCodes::NO_COMMON_TYPE); /// If there are signed and unsigned types of same bit-width, the result must be signed number with at least one more bit. /// Example, common of Int32, UInt32 = Int64. @@ -129,10 +137,9 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ if (min_bit_width_of_integer != 64) ++min_bit_width_of_integer; else - return throw_or_return( - getExceptionMessagePrefix(types) - + " because some of them are signed integers and some are unsigned integers," - " but there is no signed integer type, that can exactly represent all required unsigned integer values", + return throwOrReturn(types, + "because some of them are signed integers and some are unsigned integers," + " but there is no signed integer type, that can exactly represent all required unsigned integer values", ErrorCodes::NO_COMMON_TYPE); } @@ -145,8 +152,8 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ else if (min_mantissa_bits <= 53) return std::make_shared(); else - return throw_or_return(getExceptionMessagePrefix(types) - + " because some of them are integers and some are floating point," + return throwOrReturn(types, + " because some of them are integers and some are floating point," " but there is no floating point type, that can exactly represent all required integers", ErrorCodes::NO_COMMON_TYPE); } @@ -166,8 +173,8 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ else if (min_bit_width_of_integer <= 256) return std::make_shared(); else - return throw_or_return(getExceptionMessagePrefix(types) - + " because some of them are signed integers and some are unsigned integers," + return throwOrReturn(types, + " because some of them are signed integers and some are unsigned integers," " but there is no signed integer type, that can exactly represent all required unsigned integer values", ErrorCodes::NO_COMMON_TYPE); } @@ -186,9 +193,8 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ else if (min_bit_width_of_integer <= 256) return std::make_shared(); else - return throw_or_return("Logical error: " + getExceptionMessagePrefix(types) - + " but as all data types are unsigned integers, we must have found maximum unsigned integer type", ErrorCodes::NO_COMMON_TYPE); - + return throwOrReturn(types, + " but as all data types are unsigned integers, we must have found maximum unsigned integer type", ErrorCodes::NO_COMMON_TYPE); } } @@ -197,16 +203,9 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ } -DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_string) +template +DataTypePtr getLeastSupertype(const DataTypes & types) { - auto throw_or_return = [&](std::string_view message, int error_code) - { - if (allow_conversion_to_string) - return std::make_shared(); - - throw Exception(String(message), error_code); - }; - /// Trivial cases if (types.empty()) @@ -243,7 +242,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ non_nothing_types.emplace_back(type); if (non_nothing_types.size() < types.size()) - return getLeastSupertype(non_nothing_types, allow_conversion_to_string); + return getLeastSupertype(non_nothing_types); } /// For Arrays @@ -268,9 +267,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_array) { if (!all_arrays) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Array and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are Array and some of them are not", ErrorCodes::NO_COMMON_TYPE); - return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); + return std::make_shared(getLeastSupertype(nested_types)); } } @@ -294,7 +293,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ nested_types[elem_idx].reserve(types.size()); } else if (tuple_size != type_tuple->getElements().size()) - return throw_or_return(getExceptionMessagePrefix(types) + " because Tuples have different sizes", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because Tuples have different sizes", ErrorCodes::NO_COMMON_TYPE); have_tuple = true; @@ -308,11 +307,11 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_tuple) { if (!all_tuples) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Tuple and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are Tuple and some of them are not", ErrorCodes::NO_COMMON_TYPE); DataTypes common_tuple_types(tuple_size); for (size_t elem_idx = 0; elem_idx < tuple_size; ++elem_idx) - common_tuple_types[elem_idx] = getLeastSupertype(nested_types[elem_idx], allow_conversion_to_string); + common_tuple_types[elem_idx] = getLeastSupertype(nested_types[elem_idx]); return std::make_shared(common_tuple_types); } @@ -342,11 +341,11 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_maps) { if (!all_maps) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Maps and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are Maps and some of them are not", ErrorCodes::NO_COMMON_TYPE); return std::make_shared( - getLeastSupertype(key_types, allow_conversion_to_string), - getLeastSupertype(value_types, allow_conversion_to_string)); + getLeastSupertype(key_types), + getLeastSupertype(value_types)); } } @@ -377,9 +376,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_low_cardinality) { if (have_not_low_cardinality) - return getLeastSupertype(nested_types, allow_conversion_to_string); + return getLeastSupertype(nested_types); else - return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); + return std::make_shared(getLeastSupertype(nested_types)); } } @@ -405,7 +404,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_nullable) { - return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); + return std::make_shared(getLeastSupertype(nested_types)); } } @@ -425,7 +424,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ { bool all_strings = type_ids.size() == (have_string + have_fixed_string); if (!all_strings) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are String/FixedString and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are String/FixedString and some of them are not", ErrorCodes::NO_COMMON_TYPE); return std::make_shared(); } @@ -442,8 +441,8 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ { bool all_date_or_datetime = type_ids.size() == (have_date + have_date32 + have_datetime + have_datetime64); if (!all_date_or_datetime) - return throw_or_return(getExceptionMessagePrefix(types) - + " because some of them are Date/Date32/DateTime/DateTime64 and some of them are not", + return throwOrReturn(types, + "because some of them are Date/Date32/DateTime/DateTime64 and some of them are not", ErrorCodes::NO_COMMON_TYPE); if (have_datetime64 == 0 && have_date32 == 0) @@ -520,8 +519,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ } if (num_supported != type_ids.size()) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them have no lossless conversion to Decimal", - ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them have no lossless conversion to Decimal", ErrorCodes::NO_COMMON_TYPE); UInt32 max_scale = 0; for (const auto & type : types) @@ -543,7 +541,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ } if (min_precision > DataTypeDecimal::maxPrecision()) - return throw_or_return(getExceptionMessagePrefix(types) + " because the least supertype is Decimal(" + return throwOrReturn(types, "because the least supertype is Decimal(" + toString(min_precision) + ',' + toString(max_scale) + ')', ErrorCodes::NO_COMMON_TYPE); @@ -557,68 +555,77 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ /// For numeric types, the most complicated part. { - auto numeric_type = getNumericType(type_ids, allow_conversion_to_string); + auto numeric_type = getNumericType(type_ids); if (numeric_type) return numeric_type; } /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). - return throw_or_return(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "", ErrorCodes::NO_COMMON_TYPE); } -DataTypePtr getLeastSupertype(const TypeIndexSet & types, bool allow_conversion_to_string) +DataTypePtr getLeastSupertypeOrString(const DataTypes & types) { - auto throw_or_return = [&](std::string_view message, int error_code) - { - if (allow_conversion_to_string) - return std::make_shared(); - - throw Exception(String(message), error_code); - }; - - TypeIndexSet types_set; - for (const auto & type : types) - { - if (WhichDataType(type).isNothing()) - continue; - - if (!WhichDataType(type).isSimple()) - throw Exception(ErrorCodes::NO_COMMON_TYPE, - "Cannot get common type by type ids with parametric type {}", typeToString(type)); - - types_set.insert(type); - } - - if (types_set.empty()) - return std::make_shared(); - - if (types.contains(TypeIndex::String)) - { - if (types.size() != 1) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are String and some of them are not", ErrorCodes::NO_COMMON_TYPE); - - return std::make_shared(); - } - - /// For numeric types, the most complicated part. - auto numeric_type = getNumericType(types, allow_conversion_to_string); - if (numeric_type) - return numeric_type; - - /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). - return throw_or_return(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); + return getLeastSupertype(types); } DataTypePtr tryGetLeastSupertype(const DataTypes & types) { - try - { - return getLeastSupertype(types); - } - catch (...) - { - return nullptr; - } + return getLeastSupertype(types); } +template +DataTypePtr getLeastSupertype(const TypeIndexSet & types) +{ + if (types.empty()) + return std::make_shared(); + + if (types.size() == 1) + { + WhichDataType which(*types.begin()); + if (which.isNothing()) + return std::make_shared(); + + #define DISPATCH(TYPE) \ + if (which.idx == TypeIndex::TYPE) \ + return std::make_shared>(); /// NOLINT + + FOR_NUMERIC_TYPES(DISPATCH) + #undef DISPATCH + + if (which.isString()) + return std::make_shared(); + + return throwOrReturn(types, "because cannot get common type by type indexes with non-simple types", ErrorCodes::NO_COMMON_TYPE); + } + + if (types.contains(TypeIndex::String)) + { + bool only_string = types.size() == 2 && types.contains(TypeIndex::Nothing); + if (!only_string) + return throwOrReturn(types, "because some of them are String and some of them are not", ErrorCodes::NO_COMMON_TYPE); + + return std::make_shared(); + } + + auto numeric_type = getNumericType(types); + if (numeric_type) + return numeric_type; + + return throwOrReturn(types, "", ErrorCodes::NO_COMMON_TYPE); +} + +DataTypePtr getLeastSupertypeOrString(const TypeIndexSet & types) +{ + return getLeastSupertype(types); +} + +DataTypePtr tryGetLeastSupertype(const TypeIndexSet & types) +{ + return getLeastSupertype(types); +} + +template DataTypePtr getLeastSupertype(const DataTypes & types); +template DataTypePtr getLeastSupertype(const TypeIndexSet & types); + } diff --git a/src/DataTypes/getLeastSupertype.h b/src/DataTypes/getLeastSupertype.h index 5444bb34d06..2ef4a0e6850 100644 --- a/src/DataTypes/getLeastSupertype.h +++ b/src/DataTypes/getLeastSupertype.h @@ -1,24 +1,39 @@ #pragma once - #include - namespace DB { +enum class LeastSupertypeOnError +{ + Throw, + String, + Null, +}; + /** Get data type that covers all possible values of passed data types. - * If there is no such data type, throws an exception - * or if 'allow_conversion_to_string' is true returns String as common type. + * If there is no such data type, throws an exception. * * Examples: least common supertype for UInt8, Int8 - Int16. * Examples: there is no least common supertype for Array(UInt8), Int8. */ -DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_string = false); +template +DataTypePtr getLeastSupertype(const DataTypes & types); -using TypeIndexSet = std::unordered_set; -DataTypePtr getLeastSupertype(const TypeIndexSet & types, bool allow_conversion_to_string = false); +/// Same as above but return String type instead of throwing exception. +/// All types can be casted to String, because they can be serialized to String. +DataTypePtr getLeastSupertypeOrString(const DataTypes & types); /// Same as above but return nullptr instead of throwing exception. DataTypePtr tryGetLeastSupertype(const DataTypes & types); +using TypeIndexSet = std::unordered_set; + +template +DataTypePtr getLeastSupertype(const TypeIndexSet & types); + +DataTypePtr getLeastSupertypeOrString(const TypeIndexSet & types); + +DataTypePtr tryGetLeastSupertype(const TypeIndexSet & types); + } diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp index caedfc55f3d..7ea5dbeda83 100644 --- a/src/Databases/DDLRenamingVisitor.cpp +++ b/src/Databases/DDLRenamingVisitor.cpp @@ -19,7 +19,6 @@ namespace DB namespace ErrorCodes { extern const int WRONG_DDL_RENAMING_SETTINGS; - extern const int LOGICAL_ERROR; } namespace @@ -31,24 +30,40 @@ namespace { /// CREATE TEMPORARY TABLE String table_name = create.getTable(); - const auto & new_table_name = data.renaming_map.getNewTemporaryTableName(table_name); - if (new_table_name != table_name) - create.setTable(new_table_name); + QualifiedTableName full_table_name{DatabaseCatalog::TEMPORARY_DATABASE, table_name}; + const auto & new_table_name = data.renaming_map.getNewTableName(full_table_name); + if (new_table_name != full_table_name) + { + create.setTable(new_table_name.table); + if (new_table_name.database != DatabaseCatalog::TEMPORARY_DATABASE) + { + create.temporary = false; + create.setDatabase(new_table_name.database); + } + } } else if (create.table) { /// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW - QualifiedTableName qualified_name; - qualified_name.table = create.getTable(); - qualified_name.database = create.getDatabase(); + QualifiedTableName full_name; + full_name.table = create.getTable(); + full_name.database = create.getDatabase(); - if (!qualified_name.database.empty() && !qualified_name.table.empty()) + if (!full_name.database.empty() && !full_name.table.empty()) { - auto new_qualified_name = data.renaming_map.getNewTableName(qualified_name); - if (new_qualified_name != qualified_name) + auto new_table_name = data.renaming_map.getNewTableName(full_name); + if (new_table_name != full_name) { - create.setTable(new_qualified_name.table); - create.setDatabase(new_qualified_name.database); + create.setTable(new_table_name.table); + if (new_table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) + { + create.temporary = true; + create.setDatabase(""); + } + else + { + create.setDatabase(new_table_name.database); + } } } } @@ -291,17 +306,10 @@ void DDLRenamingVisitor::visit(ASTPtr ast, const Data & data) bool DDLRenamingVisitor::needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } -void renameDatabaseAndTableNameInCreateQuery(const ContextPtr & global_context, const DDLRenamingMap & renaming_map, ASTPtr & ast) -{ - DDLRenamingVisitor::Data data{global_context, renaming_map, ast}; - DDLRenamingVisitor::Visitor{data}.visit(ast); -} - - void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name) { if (old_table_name.table.empty() || old_table_name.database.empty() || new_table_name.table.empty() || new_table_name.database.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed"); + throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Empty names are not allowed"); auto it = old_to_new_table_names.find(old_table_name); if ((it != old_to_new_table_names.end())) @@ -321,7 +329,7 @@ void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name, void DDLRenamingMap::setNewDatabaseName(const String & old_database_name, const String & new_database_name) { if (old_database_name.empty() || new_database_name.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed"); + throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Empty names are not allowed"); auto it = old_to_new_database_names.find(old_database_name); if ((it != old_to_new_database_names.end())) @@ -351,28 +359,11 @@ QualifiedTableName DDLRenamingMap::getNewTableName(const QualifiedTableName & ol return {getNewDatabaseName(old_table_name.database), old_table_name.table}; } -void DDLRenamingMap::setNewTemporaryTableName(const String & old_table_name, const String & new_table_name) + +void renameDatabaseAndTableNameInCreateQuery(ASTPtr ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context) { - if (old_table_name.empty() || new_table_name.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed"); - - auto it = old_to_new_temporary_table_names.find(old_table_name); - if ((it != old_to_new_temporary_table_names.end())) - { - if (it->second == new_table_name) - return; - throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Wrong renaming: it's specified that temporary table {} should be renamed to {} and to {} at the same time", - backQuoteIfNeed(old_table_name), backQuoteIfNeed(it->second), backQuoteIfNeed(new_table_name)); - } - old_to_new_temporary_table_names[old_table_name] = new_table_name; -} - -const String & DDLRenamingMap::getNewTemporaryTableName(const String & old_table_name) const -{ - auto it = old_to_new_temporary_table_names.find(old_table_name); - if (it != old_to_new_temporary_table_names.end()) - return it->second; - return old_table_name; + DDLRenamingVisitor::Data data{ast, renaming_map, global_context}; + DDLRenamingVisitor::Visitor{data}.visit(ast); } } diff --git a/src/Databases/DDLRenamingVisitor.h b/src/Databases/DDLRenamingVisitor.h index 9d0f770d105..44146a8ee6b 100644 --- a/src/Databases/DDLRenamingVisitor.h +++ b/src/Databases/DDLRenamingVisitor.h @@ -17,7 +17,7 @@ class DDLRenamingMap; /// Changes names of databases or tables in a create query according to a specified renaming map. /// Does not validate AST, works a best-effort way. -void renameDatabaseAndTableNameInCreateQuery(const ContextPtr & global_context, const DDLRenamingMap & renaming_map, ASTPtr & ast); +void renameDatabaseAndTableNameInCreateQuery(ASTPtr ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context); /// Renaming map keeps information about new names of databases or tables. class DDLRenamingMap @@ -25,16 +25,13 @@ class DDLRenamingMap public: void setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name); void setNewDatabaseName(const String & old_database_name, const String & new_database_name); - void setNewTemporaryTableName(const String & old_table_name, const String & new_table_name); QualifiedTableName getNewTableName(const QualifiedTableName & old_table_name) const; const String & getNewDatabaseName(const String & old_database_name) const; - const String & getNewTemporaryTableName(const String & old_table_name) const; private: std::unordered_map old_to_new_table_names; std::unordered_map old_to_new_database_names; - std::unordered_map old_to_new_temporary_table_names; }; /// Visits ASTCreateQuery and changes names of databases or tables. @@ -43,9 +40,9 @@ class DDLRenamingVisitor public: struct Data { - ContextPtr global_context; - const DDLRenamingMap & renaming_map; ASTPtr create_query; + const DDLRenamingMap & renaming_map; + ContextPtr global_context; }; using Visitor = InDepthNodeVisitor; diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index ea887c84111..92dae025dae 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -73,7 +73,7 @@ String DatabaseAtomic::getTableDataPath(const ASTCreateQuery & query) const void DatabaseAtomic::drop(ContextPtr) { - assert(tables.empty()); + assert(TSA_SUPPRESS_WARNING_FOR_READ(tables).empty()); try { fs::remove(path_to_metadata_symlink); @@ -90,40 +90,40 @@ void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name, { assert(relative_table_path != data_path && !relative_table_path.empty()); DetachedTables not_in_use; - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); not_in_use = cleanupDetachedTables(); auto table_id = table->getStorageID(); assertDetachedTableNotInUse(table_id.uuid); - DatabaseOrdinary::attachTableUnlocked(name, table, lock); + DatabaseOrdinary::attachTableUnlocked(name, table); table_name_to_path.emplace(std::make_pair(name, relative_table_path)); } StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String & name) { DetachedTables not_in_use; - std::unique_lock lock(mutex); - auto table = DatabaseOrdinary::detachTableUnlocked(name, lock); + std::lock_guard lock(mutex); + auto table = DatabaseOrdinary::detachTableUnlocked(name); table_name_to_path.erase(name); detached_tables.emplace(table->getStorageID().uuid, table); not_in_use = cleanupDetachedTables(); //-V1001 return table; } -void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool no_delay) +void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool sync) { auto table = tryGetTable(table_name, local_context); /// Remove the inner table (if any) to avoid deadlock /// (due to attempt to execute DROP from the worker thread) if (table) - table->dropInnerTableIfAny(no_delay, local_context); + table->dropInnerTableIfAny(sync, local_context); else throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", - backQuote(database_name), backQuote(table_name)); + backQuote(getDatabaseName()), backQuote(table_name)); String table_metadata_path = getObjectMetadataPath(table_name); String table_metadata_path_drop; { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); table_metadata_path_drop = DatabaseCatalog::instance().getPathForDroppedMetadata(table->getStorageID()); auto txn = local_context->getZooKeeperMetadataTransaction(); if (txn && !local_context->isInternalSubquery()) @@ -136,7 +136,7 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// TODO better detection and recovery fs::rename(table_metadata_path, table_metadata_path_drop); /// Mark table as dropped - DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw + DatabaseOrdinary::detachTableUnlocked(table_name); /// Should never throw table_name_to_path.erase(table_name); } @@ -145,11 +145,12 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// Notify DatabaseCatalog that table was dropped. It will remove table data in background. /// Cleanup is performed outside of database to allow easily DROP DATABASE without waiting for cleanup to complete. - DatabaseCatalog::instance().enqueueDroppedTableCleanup(table->getStorageID(), table, table_metadata_path_drop, no_delay); + DatabaseCatalog::instance().enqueueDroppedTableCleanup(table->getStorageID(), table, table_metadata_path_drop, sync); } void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_name, IDatabase & to_database, const String & to_table_name, bool exchange, bool dictionary) + TSA_NO_THREAD_SAFETY_ANALYSIS /// TSA does not support conditional locking { if (typeid(*this) != typeid(to_database)) { @@ -173,7 +174,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ String old_metadata_path = getObjectMetadataPath(table_name); String new_metadata_path = to_database.getObjectMetadataPath(to_table_name); - auto detach = [](DatabaseAtomic & db, const String & table_name_, bool has_symlink) + auto detach = [](DatabaseAtomic & db, const String & table_name_, bool has_symlink) TSA_REQUIRES(db.mutex) { auto it = db.table_name_to_path.find(table_name_); String table_data_path_saved; @@ -188,7 +189,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ return table_data_path_saved; }; - auto attach = [](DatabaseAtomic & db, const String & table_name_, const String & table_data_path_, const StoragePtr & table_) + auto attach = [](DatabaseAtomic & db, const String & table_name_, const String & table_data_path_, const StoragePtr & table_) TSA_REQUIRES(db.mutex) { db.tables.emplace(table_name_, table_); if (table_data_path_.empty()) @@ -229,9 +230,9 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ } if (!exchange) - other_db.checkMetadataFilenameAvailabilityUnlocked(to_table_name, inside_database ? db_lock : other_db_lock); + other_db.checkMetadataFilenameAvailabilityUnlocked(to_table_name); - StoragePtr table = getTableUnlocked(table_name, db_lock); + StoragePtr table = getTableUnlocked(table_name); if (dictionary && !table->isDictionary()) throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables"); @@ -244,7 +245,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ StorageID other_table_new_id = StorageID::createEmpty(); if (exchange) { - other_table = other_db.getTableUnlocked(to_table_name, other_db_lock); + other_table = other_db.getTableUnlocked(to_table_name); if (dictionary && !other_table->isDictionary()) throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables"); other_table_new_id = {database_name, table_name, other_table->getStorageID().uuid}; @@ -292,19 +293,16 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora { DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); - bool locked_uuid = false; try { - std::unique_lock lock{mutex}; + std::lock_guard lock{mutex}; if (query.getDatabase() != database_name) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed to `{}`, cannot create table in `{}`", database_name, query.getDatabase()); /// Do some checks before renaming file from .tmp to .sql not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); - /// We will get en exception if some table with the same UUID exists (even if it's detached table or table from another database) - DatabaseCatalog::instance().addUUIDMapping(query.uuid); - locked_uuid = true; + chassert(DatabaseCatalog::instance().hasUUIDMapping(query.uuid)); auto txn = query_context->getZooKeeperMetadataTransaction(); if (txn && !query_context->isInternalSubquery()) @@ -315,14 +313,12 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora /// It throws if `table_metadata_path` already exists (it's possible if table was detached) renameNoReplace(table_metadata_tmp_path, table_metadata_path); /// Commit point (a sort of) - attachTableUnlocked(query.getTable(), table, lock); /// Should never throw + attachTableUnlocked(query.getTable(), table); /// Should never throw table_name_to_path.emplace(query.getTable(), table_data_path); } catch (...) { fs::remove(table_metadata_tmp_path); - if (locked_uuid) - DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; } if (table->storesDataOnDisk()) @@ -335,8 +331,8 @@ void DatabaseAtomic::commitAlterTable(const StorageID & table_id, const String & bool check_file_exists = true; SCOPE_EXIT({ std::error_code code; if (check_file_exists) std::filesystem::remove(table_metadata_tmp_path, code); }); - std::unique_lock lock{mutex}; - auto actual_table_id = getTableUnlocked(table_id.table_name, lock)->getStorageID(); + std::lock_guard lock{mutex}; + auto actual_table_id = getTableUnlocked(table_id.table_name)->getStorageID(); if (table_id.uuid != actual_table_id.uuid) throw Exception("Cannot alter table because it was renamed", ErrorCodes::CANNOT_ASSIGN_ALTER); @@ -368,7 +364,7 @@ void DatabaseAtomic::assertDetachedTableNotInUse(const UUID & uuid) void DatabaseAtomic::setDetachedTableNotInUseForce(const UUID & uuid) { - std::unique_lock lock{mutex}; + std::lock_guard lock{mutex}; detached_tables.erase(uuid); } diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index b748e53244d..6cb2226a7f8 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -35,7 +35,7 @@ public: bool exchange, bool dictionary) override; - void dropTable(ContextPtr context, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr context, const String & table_name, bool sync) override; void attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) override; StoragePtr detachTable(ContextPtr context, const String & name) override; @@ -70,9 +70,9 @@ protected: void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, const String & table_metadata_tmp_path, const String & table_metadata_path, ContextPtr query_context) override; - void assertDetachedTableNotInUse(const UUID & uuid); + void assertDetachedTableNotInUse(const UUID & uuid) TSA_REQUIRES(mutex); using DetachedTables = std::unordered_map; - [[nodiscard]] DetachedTables cleanupDetachedTables(); + [[nodiscard]] DetachedTables cleanupDetachedTables() TSA_REQUIRES(mutex); void tryCreateMetadataSymlink(); @@ -80,9 +80,9 @@ protected: //TODO store path in DatabaseWithOwnTables::tables using NameToPathMap = std::unordered_map; - NameToPathMap table_name_to_path; + NameToPathMap table_name_to_path TSA_GUARDED_BY(mutex); - DetachedTables detached_tables; + DetachedTables detached_tables TSA_GUARDED_BY(mutex); String path_to_table_symlinks; String path_to_metadata_symlink; const UUID db_uuid; diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 5cc334eaad4..af82d382063 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "config_core.h" @@ -61,36 +62,19 @@ namespace ErrorCodes DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & metadata_path, ContextPtr context) { - bool created = false; + /// Creates store/xxx/ for Atomic + fs::create_directories(fs::path(metadata_path).parent_path()); - try - { - /// Creates store/xxx/ for Atomic - fs::create_directories(fs::path(metadata_path).parent_path()); + DatabasePtr impl = getImpl(create, metadata_path, context); - /// Before 20.7 it's possible that .sql metadata file does not exist for some old database. - /// In this case Ordinary database is created on server startup if the corresponding metadata directory exists. - /// So we should remove metadata directory if database creation failed. - /// TODO remove this code - created = fs::create_directory(metadata_path); + if (impl && context->hasQueryContext() && context->getSettingsRef().log_queries) + context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Database, impl->getEngineName()); - DatabasePtr impl = getImpl(create, metadata_path, context); + /// Attach database metadata + if (impl && create.comment) + impl->setDatabaseComment(create.comment->as()->value.safeGet()); - if (impl && context->hasQueryContext() && context->getSettingsRef().log_queries) - context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Database, impl->getEngineName()); - - // Attach database metadata - if (impl && create.comment) - impl->setDatabaseComment(create.comment->as()->value.safeGet()); - - return impl; - } - catch (...) - { - if (created && fs::exists(metadata_path)) - fs::remove_all(metadata_path); - throw; - } + return impl; } template @@ -138,8 +122,14 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have table overrides", engine_name); if (engine_name == "Ordinary") + { + if (!create.attach && !context->getSettingsRef().allow_deprecated_database_ordinary) + throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, + "Ordinary database engine is deprecated (see also allow_deprecated_database_ordinary setting)"); return std::make_shared(database_name, metadata_path, context); - else if (engine_name == "Atomic") + } + + if (engine_name == "Atomic") return std::make_shared(database_name, metadata_path, uuid, context); else if (engine_name == "Memory") return std::make_shared(database_name, context); @@ -334,12 +324,25 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.username = safeGetLiteralValue(engine_args[2], engine_name); configuration.password = safeGetLiteralValue(engine_args[3], engine_name); + bool is_deprecated_syntax = false; if (engine_args.size() >= 5) - configuration.schema = safeGetLiteralValue(engine_args[4], engine_name); - } + { + auto arg_value = engine_args[4]->as()->value; + if (arg_value.getType() == Field::Types::Which::String) + { + configuration.schema = safeGetLiteralValue(engine_args[4], engine_name); + } + else + { + use_table_cache = safeGetLiteralValue(engine_args[4], engine_name); + LOG_WARNING(&Poco::Logger::get("DatabaseFactory"), "A deprecated syntax of PostgreSQL database engine is used"); + is_deprecated_syntax = true; + } + } - if (engine_args.size() >= 6) - use_table_cache = safeGetLiteralValue(engine_args[5], engine_name); + if (!is_deprecated_syntax && engine_args.size() >= 6) + use_table_cache = safeGetLiteralValue(engine_args[5], engine_name); + } auto pool = std::make_shared(configuration, context->getSettingsRef().postgresql_connection_pool_size, diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index b024c73d578..3a1b3009878 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -77,10 +77,10 @@ void DatabaseLazy::createTable( void DatabaseLazy::dropTable( ContextPtr local_context, const String & table_name, - bool no_delay) + bool sync) { SCOPE_EXIT_MEMORY_SAFE({ clearExpiredTables(); }); - DatabaseOnDisk::dropTable(local_context, table_name, no_delay); + DatabaseOnDisk::dropTable(local_context, table_name, sync); } void DatabaseLazy::renameTable( @@ -158,6 +158,7 @@ DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(ContextPtr, const Filt bool DatabaseLazy::empty() const { + std::lock_guard lock(mutex); return tables_cache.empty(); } diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 3a7d7b14be1..d3c3ed2843b 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -37,7 +37,7 @@ public: void dropTable( ContextPtr context, const String & table_name, - bool no_delay) override; + bool sync) override; void renameTable( ContextPtr context, @@ -102,8 +102,8 @@ private: const time_t expiration_time; /// TODO use DatabaseWithOwnTablesBase::tables - mutable TablesCache tables_cache; - mutable CacheExpirationQueue cache_expiration_queue; + mutable TablesCache tables_cache TSA_GUARDED_BY(mutex); + mutable CacheExpirationQueue cache_expiration_queue TSA_GUARDED_BY(mutex); StoragePtr loadTable(const String & table_name) const; diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 6df5b70c827..8540c785419 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int UNKNOWN_TABLE; extern const int LOGICAL_ERROR; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; } DatabaseMemory::DatabaseMemory(const String & name_, ContextPtr context_) @@ -32,8 +33,8 @@ void DatabaseMemory::createTable( const StoragePtr & table, const ASTPtr & query) { - std::unique_lock lock{mutex}; - attachTableUnlocked(table_name, table, lock); + std::lock_guard lock{mutex}; + attachTableUnlocked(table_name, table); /// Clean the query from temporary flags. ASTPtr query_to_store = query; @@ -52,23 +53,24 @@ void DatabaseMemory::createTable( void DatabaseMemory::dropTable( ContextPtr /*context*/, const String & table_name, - bool /*no_delay*/) + bool /*sync*/) { - std::unique_lock lock{mutex}; - auto table = detachTableUnlocked(table_name, lock); + StoragePtr table; + { + std::lock_guard lock{mutex}; + table = detachTableUnlocked(table_name); + } try { /// Remove table without lock since: /// - it does not require it /// - it may cause lock-order-inversion if underlying storage need to /// resolve tables (like StorageLiveView) - SCOPE_EXIT(lock.lock()); - lock.unlock(); table->drop(); if (table->storesDataOnDisk()) { - assert(database_name != DatabaseCatalog::TEMPORARY_DATABASE); + assert(getDatabaseName() != DatabaseCatalog::TEMPORARY_DATABASE); fs::path table_data_dir{getTableDataPath(table_name)}; if (fs::exists(table_data_dir)) fs::remove_all(table_data_dir); @@ -76,10 +78,13 @@ void DatabaseMemory::dropTable( } catch (...) { + std::lock_guard lock{mutex}; assert(database_name != DatabaseCatalog::TEMPORARY_DATABASE); - attachTableUnlocked(table_name, table, lock); + attachTableUnlocked(table_name, table); throw; } + + std::lock_guard lock{mutex}; table->is_dropped = true; create_queries.erase(table_name); UUID table_uuid = table->getStorageID().uuid; @@ -141,4 +146,42 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl DatabaseCatalog::instance().updateLoadingDependencies(table_id, std::move(new_dependencies)); } +std::vector> DatabaseMemory::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const +{ + /// We need a special processing for the temporary database. + if (getDatabaseName() != DatabaseCatalog::TEMPORARY_DATABASE) + return DatabaseWithOwnTablesBase::getTablesForBackup(filter, local_context); + + std::vector> res; + + /// `this->tables` for the temporary database doesn't contain real names of tables. + /// That's why we need to call Context::getExternalTables() and then resolve those names using tryResolveStorageID() below. + auto external_tables = local_context->getExternalTables(); + + for (const auto & [table_name, storage] : external_tables) + { + if (!filter(table_name)) + continue; + + auto storage_id = local_context->tryResolveStorageID(StorageID{"", table_name}, Context::ResolveExternal); + if (!storage_id) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't resolve the name of temporary table {}", backQuoteIfNeed(table_name)); + + /// Here `storage_id.table_name` looks like looks like "_tmp_ab9b15a3-fb43-4670-abec-14a0e9eb70f1" + /// it's not the real name of the table. + auto create_table_query = tryGetCreateTableQuery(storage_id.table_name, local_context); + if (!create_table_query) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for temporary table {}", backQuoteIfNeed(table_name)); + + const auto & create = create_table_query->as(); + if (create.getTable() != table_name) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for temporary table {}", backQuoteIfNeed(create.getTable()), backQuoteIfNeed(table_name)); + + storage->adjustCreateQueryForBackup(create_table_query); + res.emplace_back(create_table_query, storage); + } + + return res; +} + } diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index b854d9be1f3..6262543b0c1 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -32,7 +32,7 @@ public: void dropTable( ContextPtr context, const String & table_name, - bool no_delay) override; + bool sync) override; ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override; ASTPtr getCreateDatabaseQuery() const override; @@ -50,10 +50,12 @@ public: void alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; + private: - String data_path; + const String data_path; using NameToASTCreate = std::unordered_map; - NameToASTCreate create_queries; + NameToASTCreate create_queries TSA_GUARDED_BY(mutex); }; } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 9484da8ec2d..26ea3b81e3a 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -281,7 +281,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri } } -void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_name, bool /*no_delay*/) +void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_name, bool /*sync*/) { String table_metadata_path = getObjectMetadataPath(table_name); String table_metadata_path_drop = table_metadata_path + drop_suffix; @@ -321,11 +321,11 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const { - std::unique_lock lock(mutex); - checkMetadataFilenameAvailabilityUnlocked(to_table_name, lock); + std::lock_guard lock(mutex); + checkMetadataFilenameAvailabilityUnlocked(to_table_name); } -void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name, std::unique_lock &) const +void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name) const { String table_metadata_path = getObjectMetadataPath(to_table_name); @@ -395,6 +395,18 @@ void DatabaseOnDisk::renameTable( if (auto * target_db = dynamic_cast(&to_database)) target_db->checkMetadataFilenameAvailability(to_table_name); + /// This place is actually quite dangerous. Since data directory is moved to store/ + /// DatabaseCatalog may try to clean it up as unused. We add UUID mapping to avoid this. + /// However, we may fail after data directory move, but before metadata file creation in the destination db. + /// In this case nothing will protect data directory (except 30-days timeout). + /// But this situation (when table in Ordinary database is partially renamed) require manual intervention anyway. + if (from_ordinary_to_atomic) + { + DatabaseCatalog::instance().addUUIDMapping(create.uuid); + if (table->storesDataOnDisk()) + LOG_INFO(log, "Moving table from {} to {}", table_data_relative_path, to_database.getTableDataPath(create)); + } + /// Notify the table that it is renamed. It will move data to new path (if it stores data on disk) and update StorageID table->rename(to_database.getTableDataPath(create), StorageID(create)); } @@ -491,7 +503,7 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const void DatabaseOnDisk::drop(ContextPtr local_context) { - assert(tables.empty()); + assert(TSA_SUPPRESS_WARNING_FOR_READ(tables).empty()); if (local_context->getSettingsRef().force_remove_data_recursively_on_drop) { fs::remove_all(local_context->getPath() + getDataPath()); @@ -713,8 +725,6 @@ ASTPtr DatabaseOnDisk::getCreateQueryFromStorage(const String & table_name, cons void DatabaseOnDisk::modifySettingsMetadata(const SettingsChanges & settings_changes, ContextPtr query_context) { - std::lock_guard lock(modify_settings_mutex); - auto create_query = getCreateDatabaseQuery()->clone(); auto * create = create_query->as(); auto * settings = create->storage->settings; @@ -747,7 +757,7 @@ void DatabaseOnDisk::modifySettingsMetadata(const SettingsChanges & settings_cha writeChar('\n', statement_buf); String statement = statement_buf.str(); - String database_name_escaped = escapeForFileName(database_name); + String database_name_escaped = escapeForFileName(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); /// FIXME fs::path metadata_root_path = fs::canonical(query_context->getGlobalContext()->getPath()); fs::path metadata_file_tmp_path = fs::path(metadata_root_path) / "metadata" / (database_name_escaped + ".sql.tmp"); fs::path metadata_file_path = fs::path(metadata_root_path) / "metadata" / (database_name_escaped + ".sql"); diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index a118c8da678..90aba6be169 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -43,7 +43,7 @@ public: void dropTable( ContextPtr context, const String & table_name, - bool no_delay) override; + bool sync) override; void renameTable( ContextPtr context, @@ -70,7 +70,7 @@ public: /// will throw when the table we want to attach already exists (in active / detached / detached permanently form) void checkMetadataFilenameAvailability(const String & to_table_name) const override; - void checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name, std::unique_lock &) const; + void checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name) const TSA_REQUIRES(mutex); void modifySettingsMetadata(const SettingsChanges & settings_changes, ContextPtr query_context); @@ -99,9 +99,6 @@ protected: const String metadata_path; const String data_path; - - /// For alter settings. - std::mutex modify_settings_mutex; }; } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index ada9030905d..18b70222382 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -29,6 +29,12 @@ namespace fs = std::filesystem; namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; namespace @@ -83,7 +89,7 @@ void DatabaseOrdinary::loadStoredObjects( */ ParsedTablesMetadata metadata; - loadTablesMetadata(local_context, metadata); + loadTablesMetadata(local_context, metadata, force_attach); size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries; @@ -151,12 +157,12 @@ void DatabaseOrdinary::loadStoredObjects( } } -void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata) +void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata, bool is_startup) { size_t prev_tables_count = metadata.parsed_tables.size(); size_t prev_total_dictionaries = metadata.total_dictionaries; - auto process_metadata = [&metadata, this](const String & file_name) + auto process_metadata = [&metadata, is_startup, this](const String & file_name) { fs::path path(getMetadataPath()); fs::path file_path(file_name); @@ -168,20 +174,35 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables if (ast) { auto * create_query = ast->as(); - create_query->setDatabase(database_name); + /// NOTE No concurrent writes are possible during database loading + create_query->setDatabase(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); + + /// Even if we don't load the table we can still mark the uuid of it as taken. + if (create_query->uuid != UUIDHelpers::Nil) + { + /// A bit tricky way to distinguish ATTACH DATABASE and server startup (actually it's "force_attach" flag). + if (is_startup) + { + /// Server is starting up. Lock UUID used by permanently detached table. + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + } + else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + /// FIXME MaterializedPostgreSQL works with UUIDs incorrectly and breaks invariants + if (getEngineName() != "MaterializedPostgreSQL") + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } + } if (fs::exists(full_path.string() + detached_suffix)) { - /// FIXME: even if we don't load the table we can still mark the uuid of it as taken. - /// if (create_query->uuid != UUIDHelpers::Nil) - /// DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); - const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); return; } - QualifiedTableName qualified_name{database_name, create_query->getTable()}; + QualifiedTableName qualified_name{TSA_SUPPRESS_WARNING_FOR_READ(database_name), create_query->getTable()}; TableNamesSet loading_dependencies = getDependenciesSetFromCreateQuery(getContext(), qualified_name, ast); std::lock_guard lock{metadata.mutex}; @@ -214,12 +235,12 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables size_t tables_in_database = objects_in_database - dictionaries_in_database; LOG_INFO(log, "Metadata processed, database {} has {} tables and {} dictionaries in total.", - database_name, tables_in_database, dictionaries_in_database); + TSA_SUPPRESS_WARNING_FOR_READ(database_name), tables_in_database, dictionaries_in_database); } void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) { - assert(name.database == database_name); + assert(name.database == TSA_SUPPRESS_WARNING_FOR_READ(database_name)); const auto & create_query = ast->as(); tryAttachTable( @@ -235,7 +256,8 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_rest { LOG_INFO(log, "Starting up tables."); - const size_t total_tables = tables.size(); + /// NOTE No concurrent writes are possible during database loading + const size_t total_tables = TSA_SUPPRESS_WARNING_FOR_READ(tables).size(); if (!total_tables) return; @@ -251,7 +273,7 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_rest try { - for (const auto & table : tables) + for (const auto & table : TSA_SUPPRESS_WARNING_FOR_READ(tables)) thread_pool.scheduleOrThrowOnError([&]() { startup_one_table(table.second); }); } catch (...) diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 982be2024ce..6e524ae18b0 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -25,7 +25,7 @@ public: bool supportsLoadingInTopologicalOrder() const override { return true; } - void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata) override; + void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override; void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) override; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 5c701c8d90c..a7ad632efff 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -44,6 +44,7 @@ namespace ErrorCodes extern const int INCORRECT_QUERY; extern const int ALL_CONNECTION_TRIES_FAILED; extern const int NO_ACTIVE_REPLICAS; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; extern const int CANNOT_RESTORE_TABLE; } @@ -148,7 +149,7 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const if (hosts.empty()) throw Exception(ErrorCodes::NO_ACTIVE_REPLICAS, "No replicas of database {} found. " "It's possible if the first replica is not fully created yet " - "or if the last replica was just dropped or due to logical error", database_name); + "or if the last replica was just dropped or due to logical error", zookeeper_path); Int32 cversion = stat.cversion; ::sort(hosts.begin(), hosts.end()); @@ -213,7 +214,7 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const treat_local_port_as_remote, cluster_auth_info.cluster_secure_connection, /*priority=*/1, - database_name, + TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME cluster_auth_info.cluster_secret); } @@ -588,7 +589,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep query_context->makeQueryContext(); query_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; query_context->getClientInfo().is_replicated_database_internal = true; - query_context->setCurrentDatabase(database_name); + query_context->setCurrentDatabase(getDatabaseName()); query_context->setCurrentQueryId(""); auto txn = std::make_shared(current_zookeeper, zookeeper_path, false, ""); query_context->initZooKeeperMetadataTransaction(txn); @@ -608,6 +609,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep /// and make possible creation of new table with the same UUID. String query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Ordinary", backQuoteIfNeed(to_db_name)); auto query_context = Context::createCopy(getContext()); + query_context->setSetting("allow_deprecated_database_ordinary", 1); executeQuery(query, query_context, true); /// But we want to avoid discarding UUID of ReplicatedMergeTree tables, because it will not work @@ -811,7 +813,7 @@ void DatabaseReplicated::shutdown() } -void DatabaseReplicated::dropTable(ContextPtr local_context, const String & table_name, bool no_delay) +void DatabaseReplicated::dropTable(ContextPtr local_context, const String & table_name, bool sync) { auto txn = local_context->getZooKeeperMetadataTransaction(); assert(!ddl_worker->isCurrentlyActive() || txn || startsWith(table_name, ".inner_id.")); @@ -820,7 +822,7 @@ void DatabaseReplicated::dropTable(ContextPtr local_context, const String & tabl String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); txn->addOp(zkutil::makeRemoveRequest(metadata_zk_path, -1)); } - DatabaseAtomic::dropTable(local_context, table_name, no_delay); + DatabaseAtomic::dropTable(local_context, table_name, sync); } void DatabaseReplicated::renameTable(ContextPtr local_context, const String & table_name, IDatabase & to_database, @@ -922,7 +924,50 @@ String DatabaseReplicated::readMetadataFile(const String & table_name) const } -void DatabaseReplicated::createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) +std::vector> +DatabaseReplicated::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr &) const +{ + /// Here we read metadata from ZooKeeper. We could do that by simple call of DatabaseAtomic::getTablesForBackup() however + /// reading from ZooKeeper is better because thus we won't be dependent on how fast the replication queue of this database is. + std::vector> res; + auto zookeeper = getContext()->getZooKeeper(); + auto escaped_table_names = zookeeper->getChildren(zookeeper_path + "/metadata"); + for (const auto & escaped_table_name : escaped_table_names) + { + String table_name = unescapeForFileName(escaped_table_name); + if (!filter(table_name)) + continue; + String zk_metadata; + if (!zookeeper->tryGet(zookeeper_path + "/metadata/" + escaped_table_name, zk_metadata)) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Metadata for table {} was not found in ZooKeeper", table_name); + + ParserCreateQuery parser; + auto create_table_query = parseQuery(parser, zk_metadata, 0, getContext()->getSettingsRef().max_parser_depth); + + auto & create = create_table_query->as(); + create.attach = false; + create.setTable(table_name); + create.setDatabase(getDatabaseName()); + + StoragePtr storage; + if (create.uuid != UUIDHelpers::Nil) + { + storage = DatabaseCatalog::instance().tryGetByUUID(create.uuid).second; + if (storage) + storage->adjustCreateQueryForBackup(create_table_query); + } + res.emplace_back(create_table_query, storage); + } + + return res; +} + + +void DatabaseReplicated::createTableRestoredFromBackup( + const ASTPtr & create_table_query, + ContextMutablePtr local_context, + std::shared_ptr restore_coordination, + UInt64 timeout_ms) { /// Because of the replication multiple nodes can try to restore the same tables again and failed with "Table already exists" /// because of some table could be restored already on other node and then replicated to this node. @@ -930,29 +975,25 @@ void DatabaseReplicated::createTableRestoredFromBackup(const ASTPtr & create_tab /// IRestoreCoordination::acquireCreatingTableInReplicatedDatabase() and then for other nodes this function returns false which means /// this table is already being created by some other node. String table_name = create_table_query->as().getTable(); - if (restorer.getRestoreCoordination()->acquireCreatingTableInReplicatedDatabase(getZooKeeperPath(), table_name)) + if (restore_coordination->acquireCreatingTableInReplicatedDatabase(getZooKeeperPath(), table_name)) { - restorer.executeCreateQuery(create_table_query); + DatabaseAtomic::createTableRestoredFromBackup(create_table_query, local_context, restore_coordination, timeout_ms); } /// Wait until the table is actually created no matter if it's created by the current or another node and replicated to the /// current node afterwards. We have to wait because `RestorerFromBackup` is going to restore data of the table then. /// TODO: The following code doesn't look very reliable, probably we need to rewrite it somehow. - auto timeout = restorer.getTimeout(); - bool use_timeout = (timeout.count() >= 0); + auto timeout = std::chrono::milliseconds{timeout_ms}; auto start_time = std::chrono::steady_clock::now(); - while (!isTableExist(table_name, restorer.getContext())) + while (!isTableExist(table_name, local_context)) { waitForReplicaToProcessAllEntries(50); - if (use_timeout) - { - auto elapsed = std::chrono::steady_clock::now() - start_time; - if (elapsed > timeout) - throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, - "Couldn't restore table {}.{} on other node or sync it (elapsed {})", - backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(table_name), to_string(elapsed)); - } + auto elapsed = std::chrono::steady_clock::now() - start_time; + if (elapsed > timeout) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, + "Couldn't restore table {}.{} on other node or sync it (elapsed {})", + backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(table_name), to_string(elapsed)); } } diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 45a9d12981c..07014702067 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -30,7 +30,7 @@ public: String getEngineName() const override { return "Replicated"; } /// If current query is initial, then the following methods add metadata updating ZooKeeper operations to current ZooKeeperMetadataTransaction. - void dropTable(ContextPtr, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr, const String & table_name, bool sync) override; void renameTable(ContextPtr context, const String & table_name, IDatabase & to_database, const String & to_table_name, bool exchange, bool dictionary) override; void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, @@ -72,7 +72,8 @@ public: void shutdown() override; - void createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; + void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; friend struct DatabaseReplicatedTask; friend class DatabaseReplicatedDDLWorker; diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 13cd841cc6e..93a9523d115 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; extern const int CANNOT_GET_CREATE_TABLE_QUERY; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; } void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata) @@ -218,11 +219,11 @@ bool DatabaseWithOwnTablesBase::empty() const StoragePtr DatabaseWithOwnTablesBase::detachTable(ContextPtr /* context_ */, const String & table_name) { - std::unique_lock lock(mutex); - return detachTableUnlocked(table_name, lock); + std::lock_guard lock(mutex); + return detachTableUnlocked(table_name); } -StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name, std::unique_lock &) +StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name) { StoragePtr res; @@ -245,11 +246,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n void DatabaseWithOwnTablesBase::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &) { - std::unique_lock lock(mutex); - attachTableUnlocked(table_name, table, lock); + std::lock_guard lock(mutex); + attachTableUnlocked(table_name, table); } -void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, const StoragePtr & table, std::unique_lock &) +void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, const StoragePtr & table) { auto table_id = table->getStorageID(); if (table_id.database_name != database_name) @@ -313,7 +314,7 @@ DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase() } } -StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name, std::unique_lock &) const +StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name) const { auto it = tables.find(table_name); if (it != tables.end()) @@ -322,22 +323,34 @@ StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name backQuote(database_name), backQuote(table_name)); } -DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIteratorForBackup(const BackupEntriesCollector & backup_entries_collector) const +std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const { - /// Backup all the tables in this database. - /// Here we skip inner tables of materialized views. - auto skip_internal_tables = [](const String & table_name) { return !table_name.starts_with(".inner_id."); }; - return getTablesIterator(backup_entries_collector.getContext(), skip_internal_tables); + std::vector> res; + + for (auto it = getTablesIterator(local_context, filter); it->isValid(); it->next()) + { + auto create_table_query = tryGetCreateTableQuery(it->name(), local_context); + if (!create_table_query) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for table {}.{}", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(it->name())); + + const auto & create = create_table_query->as(); + if (create.getTable() != it->name()) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for table {}.{}", backQuoteIfNeed(create.getTable()), backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(it->name())); + + auto storage = it->table(); + storage->adjustCreateQueryForBackup(create_table_query); + res.emplace_back(create_table_query, storage); + } + + return res; } -void DatabaseWithOwnTablesBase::checkCreateTableQueryForBackup(const ASTPtr &, const BackupEntriesCollector &) const -{ -} - -void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) +void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr, UInt64) { /// Creates a table by executing a "CREATE TABLE" query. - restorer.executeCreateQuery(create_table_query); + InterpreterCreateQuery interpreter{create_table_query, local_context}; + interpreter.setInternal(true); + interpreter.execute(); } } diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h index fcaa4af88bb..c5842d7dac3 100644 --- a/src/Databases/DatabasesCommon.h +++ b/src/Databases/DatabasesCommon.h @@ -36,23 +36,22 @@ public: DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; - DatabaseTablesIteratorPtr getTablesIteratorForBackup(const BackupEntriesCollector & backup_entries_collector) const override; - void checkCreateTableQueryForBackup(const ASTPtr & create_table_query, const BackupEntriesCollector & backup_entries_collector) const override; - void createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; + void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; void shutdown() override; ~DatabaseWithOwnTablesBase() override; protected: - Tables tables; + Tables tables TSA_GUARDED_BY(mutex); Poco::Logger * log; DatabaseWithOwnTablesBase(const String & name_, const String & logger, ContextPtr context); - void attachTableUnlocked(const String & table_name, const StoragePtr & table, std::unique_lock & lock); - StoragePtr detachTableUnlocked(const String & table_name, std::unique_lock & lock); - StoragePtr getTableUnlocked(const String & table_name, std::unique_lock & lock) const; + void attachTableUnlocked(const String & table_name, const StoragePtr & table) TSA_REQUIRES(mutex); + StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex); + StoragePtr getTableUnlocked(const String & table_name) const TSA_REQUIRES(mutex); }; } diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index 1d5695188b7..9e33548b0dd 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -19,36 +18,18 @@ StoragePtr IDatabase::getTable(const String & name, ContextPtr context) const { if (auto storage = tryGetTable(name, context)) return storage; - throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(database_name), backQuoteIfNeed(name)); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name)); } -ASTPtr IDatabase::getCreateDatabaseQueryForBackup() const +std::vector> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const { - auto query = getCreateDatabaseQuery(); - - /// We don't want to see any UUIDs in backup (after RESTORE the table will have another UUID anyway). - auto & create = query->as(); - create.uuid = UUIDHelpers::Nil; - - return query; -} - -DatabaseTablesIteratorPtr IDatabase::getTablesIteratorForBackup(const BackupEntriesCollector &) const -{ - /// IDatabase doesn't own any tables. - return std::make_unique(Tables{}, getDatabaseName()); -} - -void IDatabase::checkCreateTableQueryForBackup(const ASTPtr & create_table_query, const BackupEntriesCollector &) const -{ - /// Cannot restore any table because IDatabase doesn't own any tables. + /// Cannot backup any table because IDatabase doesn't own any tables. throw Exception(ErrorCodes::CANNOT_BACKUP_TABLE, - "Database engine {} does not support backups, cannot backup table {}.{}", - getEngineName(), backQuoteIfNeed(getDatabaseName()), - backQuoteIfNeed(create_table_query->as().getTable())); + "Database engine {} does not support backups, cannot backup tables in database {}", + getEngineName(), backQuoteIfNeed(getDatabaseName())); } -void IDatabase::createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup &) +void IDatabase::createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr, std::shared_ptr, UInt64) { /// Cannot restore any table because IDatabase doesn't own any tables. throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 38c85cf3d05..72155bc818c 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -30,8 +30,7 @@ class SettingsChanges; using DictionariesWithID = std::vector>; struct ParsedTablesMetadata; struct QualifiedTableName; -class BackupEntriesCollector; -class RestorerFromBackup; +class IRestoreCoordination; namespace ErrorCodes { @@ -148,7 +147,7 @@ public: { } - virtual void loadTablesMetadata(ContextPtr /*local_context*/, ParsedTablesMetadata & /*metadata*/) + virtual void loadTablesMetadata(ContextPtr /*local_context*/, ParsedTablesMetadata & /*metadata*/, bool /*is_startup*/) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } @@ -198,7 +197,7 @@ public: virtual void dropTable( /// NOLINT ContextPtr /*context*/, const String & /*name*/, - [[maybe_unused]] bool no_delay = false) + [[maybe_unused]] bool sync = false) { throw Exception("There is no DROP TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } @@ -333,17 +332,11 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); } - /// Returns a slightly changed version of the CREATE DATABASE query which must be written to a backup. - virtual ASTPtr getCreateDatabaseQueryForBackup() const; - - /// Returns an iterator that passes through all the tables when an user wants to backup the whole database. - virtual DatabaseTablesIteratorPtr getTablesIteratorForBackup(const BackupEntriesCollector & restorer) const; - - /// Checks a CREATE TABLE query before it will be written to a backup. Called by IStorage::getCreateQueryForBackup(). - virtual void checkCreateTableQueryForBackup(const ASTPtr & create_table_query, const BackupEntriesCollector & backup_entries_collector) const; + /// Returns CREATE TABLE queries and corresponding tables prepared for writing to a backup. + virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context) const; /// Creates a table restored from backup. - virtual void createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer); + virtual void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr context, std::shared_ptr restore_coordination, UInt64 timeout_ms); virtual ~IDatabase() = default; @@ -356,8 +349,8 @@ protected: } mutable std::mutex mutex; - String database_name; - String comment; + String database_name TSA_GUARDED_BY(mutex); + String comment TSA_GUARDED_BY(mutex); }; using DatabasePtr = std::shared_ptr; diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 13f55eab9e2..230a0b4d4a4 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -39,7 +39,7 @@ DatabaseMaterializedMySQL::DatabaseMaterializedMySQL( void DatabaseMaterializedMySQL::rethrowExceptionIfNeeded() const { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!settings->allows_query_when_mysql_lost && exception) { @@ -59,7 +59,7 @@ void DatabaseMaterializedMySQL::rethrowExceptionIfNeeded() const void DatabaseMaterializedMySQL::setException(const std::exception_ptr & exception_) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); exception = exception_; } @@ -80,10 +80,10 @@ void DatabaseMaterializedMySQL::createTable(ContextPtr context_, const String & DatabaseAtomic::createTable(context_, name, table, query); } -void DatabaseMaterializedMySQL::dropTable(ContextPtr context_, const String & name, bool no_delay) +void DatabaseMaterializedMySQL::dropTable(ContextPtr context_, const String & name, bool sync) { checkIsInternalQuery(context_, "DROP TABLE"); - DatabaseAtomic::dropTable(context_, name, no_delay); + DatabaseAtomic::dropTable(context_, name, sync); } void DatabaseMaterializedMySQL::attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.h b/src/Databases/MySQL/DatabaseMaterializedMySQL.h index 32686784f2a..a6810f29d87 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.h @@ -52,7 +52,7 @@ public: void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override; - void dropTable(ContextPtr context_, const String & name, bool no_delay) override; + void dropTable(ContextPtr context_, const String & name, bool sync) override; void attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) override; diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 58be682bd73..95098ba9cbd 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -447,7 +447,7 @@ void DatabaseMySQL::detachTablePermanently(ContextPtr, const String & table_name table_iter->second.second->is_dropped = true; } -void DatabaseMySQL::dropTable(ContextPtr local_context, const String & table_name, bool /*no_delay*/) +void DatabaseMySQL::dropTable(ContextPtr local_context, const String & table_name, bool /*sync*/) { detachTablePermanently(local_context, table_name); } diff --git a/src/Databases/MySQL/DatabaseMySQL.h b/src/Databases/MySQL/DatabaseMySQL.h index 1ee090ecd52..542cd65c1f1 100644 --- a/src/Databases/MySQL/DatabaseMySQL.h +++ b/src/Databases/MySQL/DatabaseMySQL.h @@ -82,7 +82,7 @@ public: void detachTablePermanently(ContextPtr context, const String & table_name) override; - void dropTable(ContextPtr context, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr context, const String & table_name, bool sync) override; void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override; @@ -109,15 +109,15 @@ private: void cleanOutdatedTables(); - void fetchTablesIntoLocalCache(ContextPtr context) const; + void fetchTablesIntoLocalCache(ContextPtr context) const TSA_REQUIRES(mutex); std::map fetchTablesWithModificationTime(ContextPtr local_context) const; std::map fetchTablesColumnsList(const std::vector & tables_name, ContextPtr context) const; - void destroyLocalCacheExtraTables(const std::map & tables_with_modification_time) const; + void destroyLocalCacheExtraTables(const std::map & tables_with_modification_time) const TSA_REQUIRES(mutex); - void fetchLatestTablesStructureIntoCache(const std::map & tables_modification_time, ContextPtr context) const; + void fetchLatestTablesStructureIntoCache(const std::map & tables_modification_time, ContextPtr context) const TSA_REQUIRES(mutex); ThreadFromGlobalPool thread; }; diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index 60d2fa0d2c8..8b85d1b9a63 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -63,9 +63,9 @@ void DatabaseMaterializedPostgreSQL::startSynchronization() return; replication_handler = std::make_unique( - /* replication_identifier */database_name, + /* replication_identifier */ TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME remote_database_name, - database_name, + TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME connection_info, getContext(), is_attach, @@ -99,7 +99,8 @@ void DatabaseMaterializedPostgreSQL::startSynchronization() else { /// Nested table does not exist and will be created by replication thread. - storage = std::make_shared(StorageID(database_name, table_name), getContext(), remote_database_name, table_name); + /// FIXME TSA + storage = std::make_shared(StorageID(TSA_SUPPRESS_WARNING_FOR_READ(database_name), table_name), getContext(), remote_database_name, table_name); } /// Cache MaterializedPostgreSQL wrapper over nested table. @@ -210,7 +211,8 @@ ASTPtr DatabaseMaterializedPostgreSQL::getCreateTableQueryImpl(const String & ta std::lock_guard lock(handler_mutex); - auto storage = std::make_shared(StorageID(database_name, table_name), getContext(), remote_database_name, table_name); + /// FIXME TSA + auto storage = std::make_shared(StorageID(TSA_SUPPRESS_WARNING_FOR_READ(database_name), table_name), getContext(), remote_database_name, table_name); auto ast_storage = replication_handler->getCreateNestedTableQuery(storage.get(), table_name); assert_cast(ast_storage.get())->uuid = UUIDHelpers::generateV4(); return ast_storage; @@ -234,7 +236,7 @@ ASTPtr DatabaseMaterializedPostgreSQL::createAlterSettingsQuery(const SettingCha auto * alter = query->as(); alter->alter_object = ASTAlterQuery::AlterObjectType::DATABASE; - alter->setDatabase(database_name); + alter->setDatabase(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); /// FIXME alter->set(alter->command_list, command_list); return query; @@ -259,6 +261,7 @@ void DatabaseMaterializedPostgreSQL::createTable(ContextPtr local_context, const auto * create_query = assert_cast(query_copy.get()); create_query->attach = false; create_query->attach_short_syntax = false; + DatabaseCatalog::instance().addUUIDMapping(create->uuid); DatabaseAtomic::createTable(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), table_name, table, query_copy); /// Attach MaterializedPostgreSQL table. @@ -389,10 +392,10 @@ void DatabaseMaterializedPostgreSQL::stopReplication() } -void DatabaseMaterializedPostgreSQL::dropTable(ContextPtr local_context, const String & table_name, bool no_delay) +void DatabaseMaterializedPostgreSQL::dropTable(ContextPtr local_context, const String & table_name, bool sync) { /// Modify context into nested_context and pass query to Atomic database. - DatabaseAtomic::dropTable(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), table_name, no_delay); + DatabaseAtomic::dropTable(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), table_name, sync); } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h index 08420f4ba5e..ac2bcedca60 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h @@ -55,7 +55,7 @@ public: StoragePtr detachTable(ContextPtr context, const String & table_name) override; - void dropTable(ContextPtr local_context, const String & name, bool no_delay) override; + void dropTable(ContextPtr local_context, const String & name, bool sync) override; void drop(ContextPtr local_context) override; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 1bcc203beb9..c4b815c0c9f 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -264,7 +264,7 @@ void DatabasePostgreSQL::createTable(ContextPtr local_context, const String & ta } -void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool /* no_delay */) +void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool /* sync */) { std::lock_guard lock{mutex}; @@ -369,7 +369,11 @@ ASTPtr DatabasePostgreSQL::getCreateDatabaseQuery() const ASTPtr DatabasePostgreSQL::getCreateTableQueryImpl(const String & table_name, ContextPtr local_context, bool throw_on_error) const { - auto storage = fetchTable(table_name, local_context, false); + StoragePtr storage; + { + std::lock_guard lock{mutex}; + storage = fetchTable(table_name, local_context, false); + } if (!storage) { if (throw_on_error) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index 3397dcc8076..fe4dff2ca93 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -53,7 +53,7 @@ public: StoragePtr tryGetTable(const String & name, ContextPtr context) const override; void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override; - void dropTable(ContextPtr, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr, const String & table_name, bool sync) override; void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override; StoragePtr detachTable(ContextPtr context, const String & table_name) override; @@ -81,7 +81,7 @@ private: bool checkPostgresTable(const String & table_name) const; - StoragePtr fetchTable(const String & table_name, ContextPtr context, bool table_checked) const; + StoragePtr fetchTable(const String & table_name, ContextPtr context, bool table_checked) const TSA_REQUIRES(mutex); void removeOutdatedTables(); diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 9f136efa1ff..10cde43e9e1 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -263,10 +263,11 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) " "AND NOT attisdropped AND attnum > 0", where); - table.physical_columns = readNamesAndTypesList(tx, postgres_table, query, use_nulls, false); + auto postgres_table_with_schema = postgres_schema.empty() ? postgres_table : doubleQuoteString(postgres_schema) + '.' + doubleQuoteString(postgres_table); + table.physical_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, false); if (!table.physical_columns) - throw Exception(ErrorCodes::UNKNOWN_TABLE, "PostgreSQL table {} does not exist", postgres_table); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "PostgreSQL table {} does not exist", postgres_table_with_schema); if (with_primary_key) { @@ -278,7 +279,7 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "AND a.attnum = ANY(i.indkey) " "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) AND i.indisprimary", where); - table.primary_key_columns = readNamesAndTypesList(tx, postgres_table, query, use_nulls, true); + table.primary_key_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, true); } if (with_replica_identity_index && !table.primary_key_columns) @@ -299,11 +300,13 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "and a.attnum = ANY(ix.indkey) " "and t.relkind in ('r', 'p') " /// simple tables "and t.relname = {} " /// Connection is already done to a needed database, only table name is needed. + "{}" "and ix.indisreplident = 't' " /// index is is replica identity index - "ORDER BY a.attname", /// column names - quoteString(postgres_table)); + "ORDER BY a.attname", /// column name + (postgres_schema.empty() ? "" : "and t.relnamespace = " + quoteString(postgres_schema)) + " ", + quoteString(postgres_table)); - table.replica_identity_columns = readNamesAndTypesList(tx, postgres_table, query, use_nulls, true); + table.replica_identity_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, true); } return table; diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index 7a4844e4d69..44a392ce1f2 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -173,12 +173,16 @@ ASTPtr DatabaseSQLite::getCreateDatabaseQuery() const ASTPtr DatabaseSQLite::getCreateTableQueryImpl(const String & table_name, ContextPtr local_context, bool throw_on_error) const { - auto storage = fetchTable(table_name, local_context, false); + StoragePtr storage; + { + std::lock_guard lock(mutex); + storage = fetchTable(table_name, local_context, false); + } if (!storage) { if (throw_on_error) throw Exception(ErrorCodes::UNKNOWN_TABLE, "SQLite table {}.{} does not exist", - database_name, table_name); + getDatabaseName(), table_name); return nullptr; } auto table_storage_define = database_engine_define->clone(); diff --git a/src/Databases/SQLite/DatabaseSQLite.h b/src/Databases/SQLite/DatabaseSQLite.h index c8df79d0f6a..8f0c9b4d720 100644 --- a/src/Databases/SQLite/DatabaseSQLite.h +++ b/src/Databases/SQLite/DatabaseSQLite.h @@ -54,9 +54,9 @@ private: bool checkSQLiteTable(const String & table_name) const; - NameSet fetchTablesList() const; + NameSet fetchTablesList() const TSA_REQUIRES(mutex); - StoragePtr fetchTable(const String & table_name, ContextPtr context, bool table_checked) const; + StoragePtr fetchTable(const String & table_name, ContextPtr context, bool table_checked) const TSA_REQUIRES(mutex); }; diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index 898376f8e0f..7e9b83d423a 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -93,7 +93,7 @@ void TablesLoader::loadTables() for (auto & database_name : databases_to_load) { databases[database_name]->beforeLoadingMetadata(global_context, force_restore, force_attach); - databases[database_name]->loadTablesMetadata(global_context, metadata); + databases[database_name]->loadTablesMetadata(global_context, metadata, force_attach); } LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec", @@ -171,6 +171,11 @@ void TablesLoader::removeUnresolvableDependencies(bool remove_loaded) void TablesLoader::loadTablesInTopologicalOrder(ThreadPool & pool) { + /// Compatibility setting which should be enabled by default on attach + /// Otherwise server will be unable to start for some old-format of IPv6/IPv4 types of columns + ContextMutablePtr load_context = Context::createCopy(global_context); + load_context->setSetting("cast_ipv4_ipv6_default_on_conversion_error", 1); + /// Load independent tables in parallel. /// Then remove loaded tables from dependency graph, find tables/dictionaries that do not have unresolved dependencies anymore, /// move them to the list of independent tables and load. @@ -183,7 +188,7 @@ void TablesLoader::loadTablesInTopologicalOrder(ThreadPool & pool) assert(metadata.parsed_tables.size() == tables_processed + metadata.independent_database_objects.size() + getNumberOfTablesWithDependencies()); logDependencyGraph(); - startLoadingIndependentTables(pool, level); + startLoadingIndependentTables(pool, level, load_context); TableNames new_independent_database_objects; for (const auto & table_name : metadata.independent_database_objects) @@ -237,7 +242,7 @@ DependenciesInfosIter TablesLoader::removeResolvedDependency(const DependenciesI return metadata.dependencies_info.erase(info_it); } -void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level) +void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level, ContextMutablePtr load_context) { size_t total_tables = metadata.parsed_tables.size(); @@ -245,10 +250,10 @@ void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level for (const auto & table_name : metadata.independent_database_objects) { - pool.scheduleOrThrowOnError([this, total_tables, &table_name]() + pool.scheduleOrThrowOnError([this, load_context, total_tables, &table_name]() { const auto & path_and_query = metadata.parsed_tables[table_name]; - databases[table_name.database]->loadTableFromMetadata(global_context, path_and_query.path, table_name, path_and_query.ast, force_restore); + databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, force_restore); logAboutProgress(log, ++tables_processed, total_tables, stopwatch); }); } diff --git a/src/Databases/TablesLoader.h b/src/Databases/TablesLoader.h index a14a28c487a..43e8bfdb92c 100644 --- a/src/Databases/TablesLoader.h +++ b/src/Databases/TablesLoader.h @@ -12,7 +12,7 @@ namespace Poco { - class Logger; + class Logger; // NOLINT(cppcoreguidelines-virtual-class-destructor) } class AtomicStopwatch; @@ -104,7 +104,7 @@ private: DependenciesInfosIter removeResolvedDependency(const DependenciesInfosIter & info_it, TableNames & independent_database_objects); - void startLoadingIndependentTables(ThreadPool & pool, size_t level); + void startLoadingIndependentTables(ThreadPool & pool, size_t level, ContextMutablePtr load_context); void checkCyclicDependencies() const; diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index 0dbaf2716ba..d4716999b47 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -190,7 +190,7 @@ private: /// Dictionary source should be used with mutex mutable std::mutex source_mutex; - mutable DictionarySourcePtr source_ptr; + mutable DictionarySourcePtr source_ptr TSA_GUARDED_BY(source_mutex); CacheDictionaryStoragePtr cache_storage_ptr; mutable CacheDictionaryUpdateQueue update_queue; diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp index 2077f846f09..1fdaf10c57c 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp @@ -68,9 +68,9 @@ void CacheDictionaryUpdateQueue::waitForCurrentUpdateFinish if (update_queue.isFinished()) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "CacheDictionaryUpdateQueue finished"); - std::unique_lock update_lock(update_mutex); + std::unique_lock update_lock(update_unit_ptr->update_mutex); - bool result = is_update_finished.wait_for( + bool result = update_unit_ptr->is_update_finished.wait_for( update_lock, std::chrono::milliseconds(configuration.query_wait_timeout_milliseconds), [&] @@ -133,19 +133,23 @@ void CacheDictionaryUpdateQueue::updateThreadFunction() /// Update update_func(unit_to_update); - /// Notify thread about finished updating the bunch of ids - /// where their own ids were included. - std::unique_lock lock(update_mutex); + { + /// Notify thread about finished updating the bunch of ids + /// where their own ids were included. + std::lock_guard lock(unit_to_update->update_mutex); + unit_to_update->is_done = true; + } - unit_to_update->is_done = true; - is_update_finished.notify_all(); + unit_to_update->is_update_finished.notify_all(); } catch (...) { - std::unique_lock lock(update_mutex); + { + std::lock_guard lock(unit_to_update->update_mutex); + unit_to_update->current_exception = std::current_exception(); // NOLINT(bugprone-throw-keyword-missing) + } - unit_to_update->current_exception = std::current_exception(); // NOLINT(bugprone-throw-keyword-missing) - is_update_finished.notify_all(); + unit_to_update->is_update_finished.notify_all(); } } } diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.h b/src/Dictionaries/CacheDictionaryUpdateQueue.h index d6a195ca7b8..8db5c4a59df 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.h +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.h @@ -74,7 +74,10 @@ private: template friend class CacheDictionaryUpdateQueue; - std::atomic is_done{false}; + mutable std::mutex update_mutex; + mutable std::condition_variable is_update_finished; + + bool is_done{false}; std::exception_ptr current_exception{nullptr}; /// NOLINT /// While UpdateUnit is alive, it is accounted in update_queue size. @@ -159,9 +162,6 @@ private: UpdateQueue update_queue; ThreadPool update_pool; - - mutable std::mutex update_mutex; - mutable std::condition_variable is_update_finished; }; extern template class CacheDictionaryUpdateQueue; diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index e51caf7112a..e0cf2483b3d 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -194,12 +194,8 @@ QueryPipeline CassandraDictionarySource::loadUpdatedAll() CassSessionShared CassandraDictionarySource::getSession() { /// Reuse connection if exists, create new one if not - auto session = maybe_session.lock(); - if (session) - return session; - std::lock_guard lock(connect_mutex); - session = maybe_session.lock(); + auto session = maybe_session.lock(); if (session) return session; diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index c2038a966ea..e73383aa75c 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -82,9 +82,10 @@ private: Block sample_block; ExternalQueryBuilder query_builder; - std::mutex connect_mutex; CassClusterPtr cluster; - CassSessionWeak maybe_session; + + std::mutex connect_mutex; + CassSessionWeak maybe_session TSA_GUARDED_BY(connect_mutex); }; } diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index 480befdcfb2..3f3c60206d6 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -62,26 +62,26 @@ public: std::string getFullName() const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_id.getNameForLogs(); } StorageID getDictionaryID() const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_id; } void updateDictionaryName(const StorageID & new_name) const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; assert(new_name.uuid == dictionary_id.uuid && dictionary_id.uuid != UUIDHelpers::Nil); dictionary_id = new_name; } std::string getLoadableName() const final { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_id.getInternalDictionaryName(); } @@ -92,6 +92,8 @@ public: std::string getDatabaseOrNoDatabaseTag() const { + std::lock_guard lock{mutex}; + if (!dictionary_id.database_name.empty()) return dictionary_id.database_name; @@ -278,22 +280,20 @@ public: void setDictionaryComment(String new_comment) { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; dictionary_comment = std::move(new_comment); } String getDictionaryComment() const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_comment; } private: - mutable std::mutex name_mutex; - mutable StorageID dictionary_id; - -protected: - String dictionary_comment; + mutable std::mutex mutex; + mutable StorageID dictionary_id TSA_GUARDED_BY(mutex); + String dictionary_comment TSA_GUARDED_BY(mutex); }; } diff --git a/src/Dictionaries/RedisDictionarySource.h b/src/Dictionaries/RedisDictionarySource.h index bf745a7bb41..26f5ab2a613 100644 --- a/src/Dictionaries/RedisDictionarySource.h +++ b/src/Dictionaries/RedisDictionarySource.h @@ -8,11 +8,6 @@ namespace Poco { - namespace Util - { - class AbstractConfiguration; - } - namespace Redis { class Client; diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index 8e355f70432..45c98224966 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -90,7 +90,7 @@ DiskCacheWrapper::DiskCacheWrapper( std::shared_ptr DiskCacheWrapper::acquireDownloadMetadata(const String & path) const { - std::unique_lock lock{mutex}; + std::lock_guard lock{mutex}; auto it = file_downloads.find(path); if (it != file_downloads.end()) @@ -101,7 +101,7 @@ std::shared_ptr DiskCacheWrapper::acquireDownloadMetadata( new FileDownloadMetadata, [this, path] (FileDownloadMetadata * p) { - std::unique_lock erase_lock{mutex}; + std::lock_guard erase_lock{mutex}; file_downloads.erase(path); delete p; }); diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 08a11cd3f03..e17a5aff3c7 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -75,7 +75,7 @@ public: void startup(ContextPtr context) override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; String getCacheBasePath() const override { return delegate->getCacheBasePath(); } - std::vector getRemotePaths(const String & path) const override { return delegate->getRemotePaths(path); } + PathsWithSize getObjectStoragePaths(const String & path) const override { return delegate->getObjectStoragePaths(path); } void getRemotePathsRecursive(const String & path, std::vector & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); } MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); } diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index 674179fa4a0..99dfc8e96a0 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -318,10 +318,10 @@ String DiskRestartProxy::getCacheBasePath() const return DiskDecorator::getCacheBasePath(); } -std::vector DiskRestartProxy::getRemotePaths(const String & path) const +PathsWithSize DiskRestartProxy::getObjectStoragePaths(const String & path) const { ReadLock lock (mutex); - return DiskDecorator::getRemotePaths(path); + return DiskDecorator::getObjectStoragePaths(path); } void DiskRestartProxy::getRemotePathsRecursive(const String & path, std::vector & paths_map) diff --git a/src/Disks/DiskRestartProxy.h b/src/Disks/DiskRestartProxy.h index 52d68806ab0..e483936c817 100644 --- a/src/Disks/DiskRestartProxy.h +++ b/src/Disks/DiskRestartProxy.h @@ -65,7 +65,7 @@ public: String getUniqueId(const String & path) const override; bool checkUniqueId(const String & id) const override; String getCacheBasePath() const override; - std::vector getRemotePaths(const String & path) const override; + PathsWithSize getObjectStoragePaths(const String & path) const override; void getRemotePathsRecursive(const String & path, std::vector & paths_map) override; void restart(ContextPtr context); diff --git a/src/Disks/DiskWebServer.cpp b/src/Disks/DiskWebServer.cpp index 64c020511a4..83cfbbb0150 100644 --- a/src/Disks/DiskWebServer.cpp +++ b/src/Disks/DiskWebServer.cpp @@ -170,10 +170,10 @@ std::unique_ptr DiskWebServer::readFile(const String & p auto remote_path = fs_path.parent_path() / (escapeForFileName(fs_path.stem()) + fs_path.extension().string()); remote_path = remote_path.string().substr(url.size()); - std::vector blobs_to_read; + PathsWithSize blobs_to_read; blobs_to_read.emplace_back(remote_path, iter->second.size); - auto web_impl = std::make_unique(url, path, blobs_to_read, getContext(), read_settings); + auto web_impl = std::make_unique(url, blobs_to_read, getContext(), read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { diff --git a/src/Disks/DiskWebServer.h b/src/Disks/DiskWebServer.h index 6182c97f70b..40754e71fa0 100644 --- a/src/Disks/DiskWebServer.h +++ b/src/Disks/DiskWebServer.h @@ -169,7 +169,7 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName()); } - std::vector getRemotePaths(const String &) const override { return {}; } + PathsWithSize getObjectStoragePaths(const String &) const override { return {}; } void getRemotePathsRecursive(const String &, std::vector &) override {} diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index f441f0827fb..27cac7a5456 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -219,13 +219,13 @@ public: /// Returns a list of paths because for Log family engines there might be /// multiple files in remote fs for single clickhouse file. - virtual std::vector getRemotePaths(const String &) const + virtual PathsWithSize getObjectStoragePaths(const String &) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePaths() not implemented for disk: {}`", getType()); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getObjectStoragePaths() not implemented for disk: {}`", getType()); } /// For one local path there might be multiple remote paths in case of Log family engines. - using LocalPathWithRemotePaths = std::pair>; + using LocalPathWithRemotePaths = std::pair; virtual void getRemotePathsRecursive(const String &, std::vector &) { diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 19f3a12b38d..1a4ae3f963f 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -40,7 +40,7 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c appendFilesystemCacheLog(); } - current_file_path = fs::path(common_path_prefix) / path; + current_file_path = path; current_file_size = file_size; total_bytes_read_from_current_file = 0; @@ -50,18 +50,30 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c #if USE_AWS_S3 SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(const String & path, size_t file_size) { - auto remote_path = fs::path(common_path_prefix) / path; auto remote_file_reader_creator = [=, this]() { return std::make_unique( - client_ptr, bucket, remote_path, version_id, max_single_read_retries, - settings, /* use_external_buffer */true, /* offset */ 0, read_until_position, /* restricted_seek */true); + client_ptr, + bucket, + path, + version_id, + max_single_read_retries, + settings, + /* use_external_buffer */true, + /* offset */0, + read_until_position, + /* restricted_seek */true); }; if (with_cache) { return std::make_shared( - remote_path, settings.remote_fs_cache, remote_file_reader_creator, settings, query_id, read_until_position ? read_until_position : file_size); + path, + settings.remote_fs_cache, + remote_file_reader_creator, + settings, + query_id, + read_until_position ? read_until_position : file_size); } return remote_file_reader_creator(); @@ -72,34 +84,46 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(con #if USE_AZURE_BLOB_STORAGE SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - current_file_path = path; - return std::make_unique(blob_container_client, path, max_single_read_retries, - max_single_download_retries, settings.remote_fs_buffer_size, /* use_external_buffer */true, read_until_position); + return std::make_unique( + blob_container_client, + path, + max_single_read_retries, + max_single_download_retries, + settings.remote_fs_buffer_size, + /* use_external_buffer */true, + read_until_position); } #endif SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - current_file_path = path; - return std::make_unique(fs::path(uri) / path, context, settings, /* use_external_buffer */true, read_until_position); + return std::make_unique( + fs::path(uri) / path, + context, + settings, + /* use_external_buffer */true, + read_until_position); } #if USE_HDFS SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - return std::make_unique(hdfs_uri, fs::path(hdfs_directory) / path, config, settings.remote_fs_buffer_size); + size_t begin_of_path = path.find('/', path.find("//") + 2); + auto hdfs_path = path.substr(begin_of_path); + auto hdfs_uri = path.substr(0, begin_of_path); + LOG_TEST(log, "HDFS uri: {}, path: {}", hdfs_path, hdfs_uri); + + return std::make_unique(hdfs_uri, hdfs_path, config); } #endif ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, const ReadSettings & settings_) : ReadBuffer(nullptr, 0) - , common_path_prefix(common_path_prefix_) , blobs_to_read(blobs_to_read_) , settings(settings_) , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr ? CurrentThread::getQueryId() : "") diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index d6b1f9d9479..6a39f4add46 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -27,8 +27,7 @@ friend class ReadIndirectBufferFromRemoteFS; public: ReadBufferFromRemoteFSGather( - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, const ReadSettings & settings_); ~ReadBufferFromRemoteFSGather() override; @@ -54,9 +53,7 @@ public: protected: virtual SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) = 0; - std::string common_path_prefix; - - BlobsPathToSize blobs_to_read; + PathsWithSize blobs_to_read; ReadSettings settings; @@ -69,6 +66,8 @@ protected: String query_id; + Poco::Logger * log; + private: SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size); @@ -95,8 +94,6 @@ private: */ size_t bytes_to_ignore = 0; - Poco::Logger * log; - size_t total_bytes_read_from_current_file = 0; bool enable_cache_log = false; @@ -112,11 +109,10 @@ public: std::shared_ptr client_ptr_, const String & bucket_, const String & version_id_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, size_t max_single_read_retries_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , client_ptr(std::move(client_ptr_)) , bucket(bucket_) , version_id(version_id_) @@ -142,12 +138,11 @@ class ReadBufferFromAzureBlobStorageGather final : public ReadBufferFromRemoteFS public: ReadBufferFromAzureBlobStorageGather( std::shared_ptr blob_container_client_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, size_t max_single_read_retries_, size_t max_single_download_retries_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , blob_container_client(blob_container_client_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) @@ -169,11 +164,10 @@ class ReadBufferFromWebServerGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromWebServerGather( const String & uri_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, ContextPtr context_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , uri(uri_) , context(context_) { @@ -194,25 +188,19 @@ class ReadBufferFromHDFSGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromHDFSGather( const Poco::Util::AbstractConfiguration & config_, - const String & hdfs_uri_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , config(config_) { - const size_t begin_of_path = hdfs_uri_.find('/', hdfs_uri_.find("//") + 2); - hdfs_directory = hdfs_uri_.substr(begin_of_path); - hdfs_uri = hdfs_uri_.substr(0, begin_of_path); } SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; private: const Poco::Util::AbstractConfiguration & config; - String hdfs_uri; - String hdfs_directory; }; + #endif } diff --git a/src/Disks/IO/ReadBufferFromWebServer.h b/src/Disks/IO/ReadBufferFromWebServer.h index ea746fb75a1..1e4219d53ee 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.h +++ b/src/Disks/IO/ReadBufferFromWebServer.h @@ -19,7 +19,8 @@ class ReadBufferFromWebServer : public SeekableReadBuffer { public: explicit ReadBufferFromWebServer( - const String & url_, ContextPtr context_, + const String & url_, + ContextPtr context_, const ReadSettings & settings_ = {}, bool use_external_buffer_ = false, size_t read_until_position = 0); diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 4ea7c609a51..c25d1d7470c 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -67,16 +67,18 @@ std::unique_ptr AzureObjectStorage::readObject( /// NOLINT } std::unique_ptr AzureObjectStorage::readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings, std::optional, std::optional) const { auto settings_ptr = settings.get(); auto reader_impl = std::make_unique( - client.get(), common_path_prefix, blobs_to_read, - settings_ptr->max_single_read_retries, settings_ptr->max_single_download_retries, read_settings); + client.get(), + paths_to_read, + settings_ptr->max_single_read_retries, + settings_ptr->max_single_download_retries, + read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { @@ -111,7 +113,7 @@ std::unique_ptr AzureObjectStorage::writeObject( /// NO return std::make_unique(std::move(buffer), std::move(finalize_callback), path); } -void AzureObjectStorage::listPrefix(const std::string & path, BlobsPathToSize & children) const +void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const { auto client_ptr = client.get(); @@ -134,10 +136,10 @@ void AzureObjectStorage::removeObject(const std::string & path) throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path); } -void AzureObjectStorage::removeObjects(const std::vector & paths) +void AzureObjectStorage::removeObjects(const PathsWithSize & paths) { auto client_ptr = client.get(); - for (const auto & path : paths) + for (const auto & [path, _] : paths) { auto delete_info = client_ptr->DeleteBlob(path); if (!delete_info.Value.Deleted) @@ -151,10 +153,10 @@ void AzureObjectStorage::removeObjectIfExists(const std::string & path) auto delete_info = client_ptr->DeleteBlob(path); } -void AzureObjectStorage::removeObjectsIfExist(const std::vector & paths) +void AzureObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) { auto client_ptr = client.get(); - for (const auto & path : paths) + for (const auto & [path, _] : paths) auto delete_info = client_ptr->DeleteBlob(path); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 37c3ba72ed9..ab7d2b28508 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -59,8 +59,7 @@ public: std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & blobs_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; @@ -74,15 +73,16 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void listPrefix(const std::string & path, BlobsPathToSize & children) const override; + void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; + /// Remove file. Throws exception if file doesn't exists or it's a directory. void removeObject(const std::string & path) override; - void removeObjects(const std::vector & paths) override; + void removeObjects(const PathsWithSize & paths) override; void removeObjectIfExists(const std::string & path) override; - void removeObjectsIfExist(const std::vector & paths) override; + void removeObjectsIfExist(const PathsWithSize & paths) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; @@ -95,11 +95,19 @@ public: void startup() override {} - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; String getObjectsNamespace() const override { return ""; } - std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; + private: const String name; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 0f2c320ed67..540672d9b0a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -108,9 +108,9 @@ DiskObjectStorage::DiskObjectStorage( , metadata_helper(std::make_unique(this, ReadSettings{})) {} -std::vector DiskObjectStorage::getRemotePaths(const String & local_path) const +PathsWithSize DiskObjectStorage::getObjectStoragePaths(const String & local_path) const { - return metadata_storage->getRemotePaths(local_path); + return metadata_storage->getObjectStoragePaths(local_path); } void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector & paths_map) @@ -120,7 +120,7 @@ void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std:: { try { - paths_map.emplace_back(local_path, getRemotePaths(local_path)); + paths_map.emplace_back(local_path, getObjectStoragePaths(local_path)); } catch (const Exception & e) { @@ -244,9 +244,9 @@ String DiskObjectStorage::getUniqueId(const String & path) const { LOG_TRACE(log, "Remote path: {}, Path: {}", remote_fs_root_path, path); String id; - auto blobs_paths = metadata_storage->getRemotePaths(path); + auto blobs_paths = metadata_storage->getObjectStoragePaths(path); if (!blobs_paths.empty()) - id = blobs_paths[0]; + id = blobs_paths[0].path; return id; } @@ -438,7 +438,11 @@ std::unique_ptr DiskObjectStorage::readFile( std::optional read_hint, std::optional file_size) const { - return object_storage->readObjects(remote_fs_root_path, metadata_storage->getBlobs(path), settings, read_hint, file_size); + return object_storage->readObjects( + metadata_storage->getObjectStoragePaths(path), + settings, + read_hint, + file_size); } std::unique_ptr DiskObjectStorage::writeFile( diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index b1a1d263ede..ef29dc8f071 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -49,7 +49,7 @@ public: const String & getPath() const override { return metadata_storage->getPath(); } - std::vector getRemotePaths(const String & local_path) const override; + PathsWithSize getObjectStoragePaths(const String & local_path) const override; void getRemotePathsRecursive(const String & local_path, std::vector & paths_map) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp b/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp index 9311cb2c12a..99606a18517 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageCommon.cpp @@ -73,7 +73,7 @@ FileCachePtr getCachePtrForDisk( FileCacheSettings file_cache_settings; file_cache_settings.loadFromConfig(config, config_prefix); - auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, file_cache_settings); + auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, file_cache_settings, name); cache->initialize(); auto * log = &Poco::Logger::get("Disk(" + name + ")"); diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 6ed049b865d..a2c0c8abd36 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -11,6 +11,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_FORMAT; + extern const int LOGICAL_ERROR; } void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) @@ -26,14 +27,14 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) assertChar('\n', buf); - UInt32 remote_fs_objects_count; - readIntText(remote_fs_objects_count, buf); + UInt32 storage_objects_count; + readIntText(storage_objects_count, buf); assertChar('\t', buf); readIntText(total_size, buf); assertChar('\n', buf); - remote_fs_objects.resize(remote_fs_objects_count); + storage_objects.resize(storage_objects_count); - for (size_t i = 0; i < remote_fs_objects_count; ++i) + for (size_t i = 0; i < storage_objects_count; ++i) { String remote_fs_object_path; size_t remote_fs_object_size; @@ -50,8 +51,8 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size()); } assertChar('\n', buf); - remote_fs_objects[i].relative_path = remote_fs_object_path; - remote_fs_objects[i].bytes_size = remote_fs_object_size; + storage_objects[i].path = remote_fs_object_path; + storage_objects[i].bytes_size = remote_fs_object_size; } readIntText(ref_count, buf); @@ -75,12 +76,12 @@ void DiskObjectStorageMetadata::serialize(WriteBuffer & buf, bool sync) const writeIntText(VERSION_READ_ONLY_FLAG, buf); writeChar('\n', buf); - writeIntText(remote_fs_objects.size(), buf); + writeIntText(storage_objects.size(), buf); writeChar('\t', buf); writeIntText(total_size, buf); writeChar('\n', buf); - for (const auto & [remote_fs_object_path, remote_fs_object_size] : remote_fs_objects) + for (const auto & [remote_fs_object_path, remote_fs_object_size] : storage_objects) { writeIntText(remote_fs_object_size, buf); writeChar('\t', buf); @@ -119,8 +120,11 @@ DiskObjectStorageMetadata::DiskObjectStorageMetadata( void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { + if (!remote_fs_root_path.empty() && path.starts_with(remote_fs_root_path)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path"); + total_size += size; - remote_fs_objects.emplace_back(path, size); + storage_objects.emplace_back(path, size); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h index 27df9e57cf4..0f2a2a5507d 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h @@ -19,8 +19,8 @@ private: const std::string & common_metadata_path; - /// Remote FS objects paths and their sizes. - std::vector remote_fs_objects; + /// Relative paths of blobs. + RelativePathsWithSize storage_objects; /// URI const std::string & remote_fs_root_path; @@ -60,9 +60,9 @@ public: return remote_fs_root_path; } - std::vector getBlobs() const + RelativePathsWithSize getBlobsRelativePaths() const { - return remote_fs_objects; + return storage_objects; } bool isReadOnly() const diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index 96667b8496a..c9dbb5de078 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -84,13 +85,13 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::migrateFileToRestorableSchema { LOG_TRACE(disk->log, "Migrate file {} to restorable schema", disk->metadata_storage->getPath() + path); - auto blobs = disk->metadata_storage->getBlobs(path); - for (const auto & [key, _] : blobs) + auto objects = disk->metadata_storage->getObjectStoragePaths(path); + for (const auto & [object_path, _] : objects) { ObjectAttributes metadata { {"path", path} }; - updateObjectMetadata(disk->remote_fs_root_path + key, metadata); + updateObjectMetadata(object_path, metadata); } } void DiskObjectStorageRemoteMetadataRestoreHelper::migrateToRestorableSchemaRecursive(const String & path, Futures & results) @@ -346,7 +347,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * LOG_INFO(disk->log, "Starting restore files for disk {}", disk->name); std::vector> results; - auto restore_files = [this, &source_object_storage, &restore_information, &results](const BlobsPathToSize & keys) + auto restore_files = [this, &source_object_storage, &restore_information, &results](const PathsWithSize & keys) { std::vector keys_names; for (const auto & [key, size] : keys) @@ -379,7 +380,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * return true; }; - BlobsPathToSize children; + RelativePathsWithSize children; source_object_storage->listPrefix(restore_information.source_path, children); restore_files(children); @@ -456,7 +457,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject bool send_metadata = source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->remote_fs_root_path != restore_information.source_path; std::set renames; - auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const BlobsPathToSize & keys) + auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const PathsWithSize & keys) { const String rename = "rename"; const String hardlink = "hardlink"; @@ -523,7 +524,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject return true; }; - BlobsPathToSize children; + RelativePathsWithSize children; source_object_storage->listPrefix(restore_information.source_path + "operations/", children); restore_file_operations(children); diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index cd43ea81be0..1d16012437a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -65,7 +65,7 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation std::string path; bool delete_metadata_only; bool remove_from_cache{false}; - std::vector paths_to_remove; + PathsWithSize paths_to_remove; bool if_exists; RemoveObjectStorageOperation( @@ -96,13 +96,13 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation try { uint32_t hardlink_count = metadata_storage.getHardlinkCount(path); - auto remote_objects = metadata_storage.getRemotePaths(path); + auto objects = metadata_storage.getObjectStoragePaths(path); tx->unlinkMetadata(path); if (hardlink_count == 0) { - paths_to_remove = remote_objects; + paths_to_remove = objects; remove_from_cache = true; } } @@ -134,7 +134,7 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation if (remove_from_cache) { for (const auto & path_to_remove : paths_to_remove) - object_storage.removeFromCache(path_to_remove); + object_storage.removeFromCache(path_to_remove.path); } } @@ -143,10 +143,10 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOperation { std::string path; - std::unordered_map> paths_to_remove; + std::unordered_map paths_to_remove; bool keep_all_batch_data; NameSet file_names_remove_metadata_only; - std::vector path_to_remove_from_cache; + PathsWithSize path_to_remove_from_cache; RemoveRecursiveObjectStorageOperation( IObjectStorage & object_storage_, @@ -169,14 +169,14 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp try { uint32_t hardlink_count = metadata_storage.getHardlinkCount(path_to_remove); - auto remote_objects = metadata_storage.getRemotePaths(path_to_remove); + auto objects_paths = metadata_storage.getObjectStoragePaths(path_to_remove); tx->unlinkMetadata(path_to_remove); if (hardlink_count == 0) { - paths_to_remove[path_to_remove] = remote_objects; - path_to_remove_from_cache.insert(path_to_remove_from_cache.end(), remote_objects.begin(), remote_objects.end()); + paths_to_remove[path_to_remove] = objects_paths; + path_to_remove_from_cache.insert(path_to_remove_from_cache.end(), objects_paths.begin(), objects_paths.end()); } } @@ -217,7 +217,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { if (!keep_all_batch_data) { - std::vector remove_from_remote; + PathsWithSize remove_from_remote; for (auto && [local_path, remote_paths] : paths_to_remove) { if (!file_names_remove_metadata_only.contains(fs::path(local_path).filename())) @@ -228,7 +228,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp object_storage.removeObjects(remove_from_remote); } - for (const auto & path_to_remove : path_to_remove_from_cache) + for (const auto & [path_to_remove, _] : path_to_remove_from_cache) object_storage.removeFromCache(path_to_remove); } }; @@ -238,7 +238,7 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat { std::string path_from; std::string path_to; - std::vector blobs_to_remove; + PathsWithSize blobs_to_remove; ReplaceFileObjectStorageOperation( IObjectStorage & object_storage_, @@ -254,7 +254,7 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat { if (metadata_storage.exists(path_to)) { - blobs_to_remove = metadata_storage.getRemotePaths(path_to); + blobs_to_remove = metadata_storage.getObjectStoragePaths(path_to); tx->replaceFile(path_from, path_to); } else @@ -328,14 +328,15 @@ struct CopyFileObjectStorageOperation final : public IDiskObjectStorageOperation void execute(MetadataTransactionPtr tx) override { tx->createEmptyMetadataFile(to_path); - auto source_blobs = metadata_storage.getBlobs(from_path); + auto source_blobs = metadata_storage.getObjectStoragePaths(from_path); /// Full paths + for (const auto & [blob_from, size] : source_blobs) { auto blob_name = getRandomASCIIString(); auto blob_to = fs::path(remote_fs_root_path) / blob_name; - object_storage.copyObject(fs::path(remote_fs_root_path) / blob_from, blob_to); + object_storage.copyObject(blob_from, blob_to); tx->addBlobToMetadata(to_path, blob_name, size); diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index 4574b8cb52c..bedd1a83df1 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -49,13 +49,12 @@ std::unique_ptr HDFSObjectStorage::readObject( /// NOLINT } std::unique_ptr HDFSObjectStorage::readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings, std::optional, std::optional) const { - auto hdfs_impl = std::make_unique(config, common_path_prefix, common_path_prefix, blobs_to_read, read_settings); + auto hdfs_impl = std::make_unique(config, paths_to_read, read_settings); auto buf = std::make_unique(std::move(hdfs_impl)); return std::make_unique(std::move(buf), settings->min_bytes_for_seek); } @@ -69,7 +68,9 @@ std::unique_ptr HDFSObjectStorage::writeObject( /// NOL const WriteSettings &) { if (attributes.has_value()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS API doesn't support custom attributes/metadata for stored objects"); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "HDFS API doesn't support custom attributes/metadata for stored objects"); /// Single O_WRONLY in libhdfs adds O_TRUNC auto hdfs_buffer = std::make_unique( @@ -80,7 +81,7 @@ std::unique_ptr HDFSObjectStorage::writeObject( /// NOL } -void HDFSObjectStorage::listPrefix(const std::string & path, BlobsPathToSize & children) const +void HDFSObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const { const size_t begin_of_path = path.find('/', path.find("//") + 2); int32_t num_entries; @@ -104,10 +105,10 @@ void HDFSObjectStorage::removeObject(const std::string & path) } -void HDFSObjectStorage::removeObjects(const std::vector & paths) +void HDFSObjectStorage::removeObjects(const PathsWithSize & paths) { - for (const auto & hdfs_path : paths) - removeObject(hdfs_path); + for (const auto & [path, _] : paths) + removeObject(path); } void HDFSObjectStorage::removeObjectIfExists(const std::string & path) @@ -116,15 +117,17 @@ void HDFSObjectStorage::removeObjectIfExists(const std::string & path) removeObject(path); } -void HDFSObjectStorage::removeObjectsIfExist(const std::vector & paths) +void HDFSObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) { - for (const auto & hdfs_path : paths) - removeObjectIfExists(hdfs_path); + for (const auto & [path, _] : paths) + removeObjectIfExists(path); } ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const { - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS API doesn't support custom attributes/metadata for stored objects"); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "HDFS API doesn't support custom attributes/metadata for stored objects"); } void HDFSObjectStorage::copyObject( /// NOLINT @@ -133,7 +136,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT std::optional object_to_attributes) { if (object_to_attributes.has_value()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS API doesn't support custom attributes/metadata for stored objects"); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "HDFS API doesn't support custom attributes/metadata for stored objects"); auto in = readObject(object_from); auto out = writeObject(object_to, WriteMode::Rewrite); diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index a9a223a3d7e..69878568548 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -61,8 +61,7 @@ public: std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; @@ -76,15 +75,16 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void listPrefix(const std::string & path, BlobsPathToSize & children) const override; + void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; + /// Remove file. Throws exception if file doesn't exists or it's a directory. void removeObject(const std::string & path) override; - void removeObjects(const std::vector & paths) override; + void removeObjects(const PathsWithSize & paths) override; void removeObjectIfExists(const std::string & path) override; - void removeObjectsIfExist(const std::vector & paths) override; + void removeObjectsIfExist(const PathsWithSize & paths) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; @@ -97,11 +97,18 @@ public: void startup() override; - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; String getObjectsNamespace() const override { return ""; } - std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; private: const Poco::Util::AbstractConfiguration & config; @@ -110,8 +117,6 @@ private: HDFSFSPtr hdfs_fs; SettingsPtr settings; - - }; } diff --git a/src/Disks/ObjectStorages/IMetadataStorage.h b/src/Disks/ObjectStorages/IMetadataStorage.h index fbf611df1ac..49f4d4bcc51 100644 --- a/src/Disks/ObjectStorages/IMetadataStorage.h +++ b/src/Disks/ObjectStorages/IMetadataStorage.h @@ -119,11 +119,9 @@ public: /// Read multiple metadata files into strings and return mapping from file_path -> metadata virtual std::unordered_map getSerializedMetadata(const std::vector & file_paths) const = 0; - /// Return list of paths corresponding to metadata stored in local path - virtual std::vector getRemotePaths(const std::string & path) const = 0; - - /// Return [(remote_path, size_in_bytes), ...] for metadata path - virtual BlobsPathToSize getBlobs(const std::string & path) const = 0; + /// Return [(object_storage_path, size_in_bytes), ...] for metadata path + /// object_storage_path is a full path to the blob. + virtual PathsWithSize getObjectStoragePaths(const std::string & path) const = 0; }; using MetadataStoragePtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 4921059c6b7..7532f2a3267 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -25,23 +25,23 @@ class WriteBufferFromFileBase; using ObjectAttributes = std::map; -/// Path to blob with it's size -struct BlobPathWithSize +/// Path to a file (always absolute) and its size. +struct PathWithSize { - std::string relative_path; + std::string path; uint64_t bytes_size; - BlobPathWithSize() = default; - BlobPathWithSize(const BlobPathWithSize & other) = default; + PathWithSize() = default; - BlobPathWithSize(const std::string & relative_path_, uint64_t bytes_size_) - : relative_path(relative_path_) + PathWithSize(const std::string & path_, uint64_t bytes_size_) + : path(path_) , bytes_size(bytes_size_) {} }; -/// List of blobs with their sizes -using BlobsPathToSize = std::vector; +/// List of paths with their sizes +using PathsWithSize = std::vector; +using RelativePathsWithSize = PathsWithSize; struct ObjectMetadata { @@ -65,8 +65,8 @@ public: /// Path exists or not virtual bool exists(const std::string & path) const = 0; - /// List on prefix, return children with their sizes. - virtual void listPrefix(const std::string & path, BlobsPathToSize & children) const = 0; + /// List on prefix, return children (relative paths) with their sizes. + virtual void listPrefix(const std::string & path, RelativePathsWithSize & children) const = 0; /// Get object metadata if supported. It should be possible to receive /// at least size of object @@ -81,8 +81,7 @@ public: /// Read multiple objects with common prefix virtual std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const = 0; @@ -101,13 +100,13 @@ public: /// Remove multiple objects. Some object storages can do batch remove in a more /// optimal way. - virtual void removeObjects(const std::vector & paths) = 0; + virtual void removeObjects(const PathsWithSize & paths) = 0; /// Remove object on path if exists virtual void removeObjectIfExists(const std::string & path) = 0; /// Remove objects on path if exists - virtual void removeObjectsIfExist(const std::vector & paths) = 0; + virtual void removeObjectsIfExist(const PathsWithSize & paths) = 0; /// Copy object with different attributes if required virtual void copyObject( /// NOLINT @@ -140,7 +139,10 @@ public: void removeFromCache(const std::string & path); /// Apply new settings, in most cases reiniatilize client and some other staff - virtual void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) = 0; + virtual void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) = 0; /// Sometimes object storages have something similar to chroot or namespace, for example /// buckets in S3. If object storage doesn't have any namepaces return empty string. @@ -148,7 +150,10 @@ public: /// FIXME: confusing function required for a very specific case. Create new instance of object storage /// in different namespace. - virtual std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) = 0; + virtual std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, ContextPtr context) = 0; protected: FileCachePtr cache; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index ae87e8c61c0..22502a6e2d1 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -56,7 +56,7 @@ void MetadataStorageFromDiskTransaction::commit() toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); { - std::unique_lock lock(metadata_storage.metadata_mutex); + std::lock_guard lock(metadata_storage.metadata_mutex); for (size_t i = 0; i < operations.size(); ++i) { try @@ -300,18 +300,21 @@ MetadataTransactionPtr MetadataStorageFromDisk::createTransaction() const return std::make_shared(*this); } -std::vector MetadataStorageFromDisk::getRemotePaths(const std::string & path) const +PathsWithSize MetadataStorageFromDisk::getObjectStoragePaths(const std::string & path) const { auto metadata = readMetadata(path); - std::vector remote_paths; - auto blobs = metadata->getBlobs(); - auto root_path = metadata->getBlobsCommonPrefix(); - remote_paths.reserve(blobs.size()); - for (const auto & [remote_path, _] : blobs) - remote_paths.push_back(fs::path(root_path) / remote_path); + auto object_storage_relative_paths = metadata->getBlobsRelativePaths(); /// Relative paths. + fs::path root_path = metadata->getBlobsCommonPrefix(); - return remote_paths; + PathsWithSize object_storage_paths; + object_storage_paths.reserve(object_storage_relative_paths.size()); + + /// Relative paths -> absolute. + for (auto & [object_relative_path, size] : object_storage_relative_paths) + object_storage_paths.emplace_back(root_path / object_relative_path, size); + + return object_storage_paths; } uint32_t MetadataStorageFromDisk::getHardlinkCount(const std::string & path) const @@ -320,12 +323,6 @@ uint32_t MetadataStorageFromDisk::getHardlinkCount(const std::string & path) con return metadata->getRefCount(); } -BlobsPathToSize MetadataStorageFromDisk::getBlobs(const std::string & path) const -{ - auto metadata = readMetadata(path); - return metadata->getBlobs(); -} - void MetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path) { auto metadata = metadata_storage.readMetadata(path); diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.h b/src/Disks/ObjectStorages/MetadataStorageFromDisk.h index 1ac68e193f2..8dd47d18fdb 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.h @@ -59,9 +59,7 @@ public: std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override; - BlobsPathToSize getBlobs(const std::string & path) const override; - - std::vector getRemotePaths(const std::string & path) const override; + PathsWithSize getObjectStoragePaths(const std::string & path) const override; uint32_t getHardlinkCount(const std::string & path) const override; diff --git a/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp b/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp index 983e36489e6..109ccf0eba7 100644 --- a/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp @@ -28,7 +28,7 @@ ClientConfigurationPerRequest ProxyResolverConfiguration::getConfiguration(const { LOG_DEBUG(&Poco::Logger::get("AWSClient"), "Obtain proxy using resolver: {}", endpoint.toString()); - std::unique_lock lock(cache_mutex); + std::lock_guard lock(cache_mutex); std::chrono::time_point now = std::chrono::system_clock::now(); @@ -110,7 +110,7 @@ void ProxyResolverConfiguration::errorReport(const ClientConfigurationPerRequest if (config.proxy_host.empty()) return; - std::unique_lock lock(cache_mutex); + std::lock_guard lock(cache_mutex); if (!cache_ttl.count() || !cache_valid) return; diff --git a/src/Disks/ObjectStorages/S3/S3Capabilities.h b/src/Disks/ObjectStorages/S3/S3Capabilities.h index 46e647da89e..622558f5d23 100644 --- a/src/Disks/ObjectStorages/S3/S3Capabilities.h +++ b/src/Disks/ObjectStorages/S3/S3Capabilities.h @@ -12,10 +12,9 @@ struct S3Capabilities { /// Google S3 implementation doesn't support batch delete /// TODO: possibly we have to use Google SDK https://github.com/googleapis/google-cloud-cpp/tree/main/google/cloud/storage - /// because looks like it miss a lot of features like: - /// 1) batch delete - /// 2) list_v2 - /// 3) multipart upload works differently + /// because looks like it misses some features: + /// 1) batch delete (DeleteObjects) + /// 2) upload part copy (UploadPartCopy) bool support_batch_delete{true}; /// Y.Cloud S3 implementation support proxy for connection diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index ffe4d2dd942..9236cde6e93 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -109,8 +109,7 @@ bool S3ObjectStorage::exists(const std::string & path) const std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings, std::optional, std::optional) const @@ -128,8 +127,12 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT auto settings_ptr = s3_settings.get(); auto s3_impl = std::make_unique( - client.get(), bucket, version_id, common_path_prefix, blobs_to_read, - settings_ptr->s3_settings.max_single_read_retries, disk_read_settings); + client.get(), + bucket, + version_id, + paths_to_read, + settings_ptr->s3_settings.max_single_read_retries, + disk_read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { @@ -192,7 +195,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN return std::make_unique(std::move(s3_buffer), std::move(finalize_callback), path); } -void S3ObjectStorage::listPrefix(const std::string & path, BlobsPathToSize & children) const +void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const { auto settings_ptr = s3_settings.get(); auto client_ptr = client.get(); @@ -253,14 +256,14 @@ void S3ObjectStorage::removeObjectImpl(const std::string & path, bool if_exists) } } -void S3ObjectStorage::removeObjectsImpl(const std::vector & paths, bool if_exists) +void S3ObjectStorage::removeObjectsImpl(const PathsWithSize & paths, bool if_exists) { if (paths.empty()) return; if (!s3_capabilities.support_batch_delete) { - for (const auto & path : paths) + for (const auto & [path, _] : paths) removeObjectImpl(path, if_exists); } else @@ -278,12 +281,12 @@ void S3ObjectStorage::removeObjectsImpl(const std::vector & paths, for (; current_position < paths.size() && current_chunk.size() < chunk_size_limit; ++current_position) { Aws::S3::Model::ObjectIdentifier obj; - obj.SetKey(paths[current_position]); + obj.SetKey(paths[current_position].path); current_chunk.push_back(obj); if (!keys.empty()) keys += ", "; - keys += paths[current_position]; + keys += paths[current_position].path; } Aws::S3::Model::Delete delkeys; @@ -308,12 +311,12 @@ void S3ObjectStorage::removeObjectIfExists(const std::string & path) removeObjectImpl(path, true); } -void S3ObjectStorage::removeObjects(const std::vector & paths) +void S3ObjectStorage::removeObjects(const PathsWithSize & paths) { removeObjectsImpl(paths, false); } -void S3ObjectStorage::removeObjectsIfExist(const std::vector & paths) +void S3ObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) { removeObjectsImpl(paths, true); } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 16e6c9c8cd5..5d4300bffd3 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -66,8 +66,7 @@ public: std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; @@ -81,15 +80,16 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void listPrefix(const std::string & path, BlobsPathToSize & children) const override; + void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; + /// Remove file. Throws exception if file doesn't exist or it's a directory. void removeObject(const std::string & path) override; - void removeObjects(const std::vector & paths) override; + void removeObjects(const PathsWithSize & paths) override; void removeObjectIfExists(const std::string & path) override; - void removeObjectsIfExist(const std::vector & paths) override; + void removeObjectsIfExist(const PathsWithSize & paths) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; @@ -108,26 +108,42 @@ public: void startup() override; - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; String getObjectsNamespace() const override { return bucket; } - std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; + private: void setNewSettings(std::unique_ptr && s3_settings_); void setNewClient(std::unique_ptr && client_); - void copyObjectImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + void copyObjectImpl( + const String & src_bucket, + const String & src_key, + const String & dst_bucket, + const String & dst_key, std::optional head = std::nullopt, std::optional metadata = std::nullopt) const; - void copyObjectMultipartImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + void copyObjectMultipartImpl( + const String & src_bucket, + const String & src_key, + const String & dst_bucket, + const String & dst_key, std::optional head = std::nullopt, std::optional metadata = std::nullopt) const; void removeObjectImpl(const std::string & path, bool if_exists); - void removeObjectsImpl(const std::vector & paths, bool if_exists); + void removeObjectsImpl(const PathsWithSize & paths, bool if_exists); Aws::S3::Model::HeadObjectOutcome requestObjectHeadData(const std::string & bucket_from, const std::string & key) const; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 7e0ce001405..0032aea57e4 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -36,7 +36,7 @@ struct FormatSettings bool seekable_read = true; UInt64 max_rows_to_read_for_schema_inference = 100; - String column_names_for_schema_inference = ""; + String column_names_for_schema_inference; enum class DateTimeInputFormat { diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 60386908f01..1f728489b73 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -72,6 +72,10 @@ if (TARGET ch_contrib::llvm) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm) endif () +if (TARGET ch_contrib::base-x) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base-x) +endif() + if (TARGET ch_contrib::base64) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base64) endif() @@ -86,8 +90,8 @@ if (TARGET ch_contrib::h3) target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3) endif() -if (TARGET ch_contrib::hyperscan) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::hyperscan) +if (TARGET ch_contrib::vectorscan) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::vectorscan) endif() if (TARGET ch_contrib::simdjson) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h new file mode 100644 index 00000000000..6979ce849f4 --- /dev/null +++ b/src/Functions/FunctionBase58Conversion.h @@ -0,0 +1,256 @@ +#pragma once +#include "config_functions.h" + +#if USE_BASEX +# include +# include +# include +# include +# include +# include +# include +# include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +struct Base58Encode +{ + static constexpr auto name = "base58Encode"; + + static void process(const ColumnString & input, ColumnString::MutablePtr & dst_column, const std::string & alphabet, size_t input_rows_count) + { + auto & dst_data = dst_column->getChars(); + auto & dst_offsets = dst_column->getOffsets(); + + /// Wikipedia states Base58 has efficiency of 73%, and we take 1.5 scale to avoid reallocation in most cases + size_t current_allocated_size = ceil(1.5 * input.getChars().size()); + + dst_data.resize(current_allocated_size); + dst_offsets.resize(input_rows_count); + + const ColumnString::Offsets & src_offsets = input.getOffsets(); + + const auto * source = input.getChars().raw_data(); + auto * dst = dst_data.data(); + auto * dst_pos = dst; + + size_t src_offset_prev = 0; + size_t processed_size = 0; + + const auto& encoder = (alphabet == "bitcoin") ? Base58::bitcoin() : + ((alphabet == "flickr") ? Base58::flickr() : + ((alphabet == "ripple") ? Base58::ripple() : + Base58::base58())); //GMP + + std::string encoded; + for (size_t row = 0; row < input_rows_count; ++row) + { + size_t srclen = src_offsets[row] - src_offset_prev - 1; + /// Why we didn't use char* here? + /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), + /// so we may need to do many resizes (the worst case -- we'll do it for each row) + /// This way we do exponential resizes and one final resize after whole operation is complete + encoded.clear(); + if (srclen) + try + { + encoder.encode(encoded, source, srclen); + } + catch (const std::invalid_argument& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } + catch (const std::domain_error& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } + size_t outlen = encoded.size(); + + if (processed_size + outlen >= current_allocated_size) + { + current_allocated_size += current_allocated_size; + dst_data.resize(current_allocated_size); + auto processed_offset = dst_pos - dst; + dst = dst_data.data(); + dst_pos = dst; + dst_pos += processed_offset; + } + std::memcpy(dst_pos, encoded.c_str(), ++outlen); + + source += srclen + 1; + dst_pos += outlen; + + dst_offsets[row] = dst_pos - dst; + src_offset_prev = src_offsets[row]; + processed_size += outlen; + } + + dst_data.resize(dst_pos - dst); + } +}; + +struct Base58Decode +{ + static constexpr auto name = "base58Decode"; + + static void process(const ColumnString & input, ColumnString::MutablePtr & dst_column, const std::string & alphabet, size_t input_rows_count) + { + auto & dst_data = dst_column->getChars(); + auto & dst_offsets = dst_column->getOffsets(); + + /// We allocate probably even more then needed to avoid many resizes + size_t current_allocated_size = input.getChars().size(); + + dst_data.resize(current_allocated_size); + dst_offsets.resize(input_rows_count); + + const ColumnString::Offsets & src_offsets = input.getOffsets(); + + const auto * source = input.getChars().raw_data(); + auto * dst = dst_data.data(); + auto * dst_pos = dst; + + size_t src_offset_prev = 0; + size_t processed_size = 0; + + const auto& decoder = (alphabet == "bitcoin") ? Base58::bitcoin() : + ((alphabet == "flickr") ? Base58::flickr() : + ((alphabet == "ripple") ? Base58::ripple() : + Base58::base58())); + + std::string decoded; + for (size_t row = 0; row < input_rows_count; ++row) + { + size_t srclen = src_offsets[row] - src_offset_prev - 1; + /// Why we didn't use char* here? + /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), + /// so we may need to do many resizes (the worst case -- we'll do it for each row) + /// This way we do exponential resizes and one final resize after whole operation is complete + decoded.clear(); + if (srclen) + try + { + decoder.decode(decoded, source, srclen); + } + catch (const std::invalid_argument& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } + catch (const std::domain_error& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } + size_t outlen = decoded.size(); + + if (processed_size + outlen >= current_allocated_size) + { + current_allocated_size += current_allocated_size; + dst_data.resize(current_allocated_size); + auto processed_offset = dst_pos - dst; + dst = dst_data.data(); + dst_pos = dst; + dst_pos += processed_offset; + } + std::memcpy(dst_pos, decoded.c_str(), ++outlen); + + source += srclen + 1; + dst_pos += outlen; + + dst_offsets[row] = dst_pos - dst; + src_offset_prev = src_offsets[row]; + processed_size += outlen; + } + + dst_data.resize(dst_pos - dst); + } +}; + +template +class FunctionBase58Conversion : public IFunction +{ +public: + static constexpr auto name = Func::name; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + String getName() const override + { + return Func::name; + } + + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1 && arguments.size() != 2) + throw Exception( + "Wrong number of arguments for function " + getName() + ": 1 or 2 expected.", + ErrorCodes::BAD_ARGUMENTS); + + if (!isString(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + ". Must be String.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (arguments.size() == 2 && !isString(arguments[1].type)) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName() + ". Must be String.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const ColumnPtr column_string = arguments[0].column; + const ColumnString * input = checkAndGetColumn(column_string.get()); + if (!input) + throw Exception( + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName() + ", must be String", + ErrorCodes::ILLEGAL_COLUMN); + + std::string alphabet = "bitcoin"; + + if (arguments.size() == 2) + { + const auto * alphabet_column = checkAndGetColumn(arguments[1].column.get()); + + if (!alphabet_column) + throw Exception("Second argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + alphabet = alphabet_column->getValue(); + if (alphabet != "bitcoin" && alphabet != "ripple" && alphabet != "flickr" && alphabet != "gmp") + throw Exception("Second argument for function " + getName() + " must be 'bitcoin', 'ripple', 'gmp' or 'flickr'", ErrorCodes::ILLEGAL_COLUMN); + + } + + auto dst_column = ColumnString::create(); + + Func::process(*input, dst_column, alphabet, input_rows_count); + + return dst_column; + } +}; +} + +#endif diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index a1d6b966660..87a3309f7ef 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -85,7 +85,7 @@ public: { if (!WhichDataType(arguments[0].type).isString()) throw Exception( - "Illegal type " + arguments[0].type->getName() + " of 1 argument of function " + getName() + ". Must be String.", + "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + ". Must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); @@ -98,7 +98,7 @@ public: if (!input) throw Exception( - "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName() + ", must be of type String", ErrorCodes::ILLEGAL_COLUMN); auto dst_column = ColumnString::create(); diff --git a/src/Functions/FunctionsBase58.cpp b/src/Functions/FunctionsBase58.cpp new file mode 100644 index 00000000000..dc325ff8cfc --- /dev/null +++ b/src/Functions/FunctionsBase58.cpp @@ -0,0 +1,17 @@ +#include +#if USE_BASEX +#include + +namespace DB +{ +void registerFunctionBase58Encode(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +void registerFunctionBase58Decode(FunctionFactory & factory) +{ + factory.registerFunction>(); +} +} +#endif diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 038bb9bf3df..e0c42401207 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -205,7 +205,7 @@ struct ConvertImpl if constexpr (std::is_same_v != std::is_same_v) { - throw Exception("Conversion between numeric types and UUID is not supported", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED); } else { diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index be295186943..392cbac8a1b 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -545,6 +545,8 @@ ColumnPtr FunctionAnyArityLogical::executeShortCircuit(ColumnsWithTy if (Name::name != NameAnd::name && Name::name != NameOr::name) throw Exception("Function " + getName() + " doesn't support short circuit execution", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + executeColumnIfNeeded(arguments[0]); + /// Let's denote x_i' = maskedExecute(x_i, mask). /// 1) AND(x_0, x_1, x_2, ..., x_n) /// We will support mask_i = x_0 & x_1 & ... & x_i. diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index e082cfbec2f..865a5d182c8 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -10,10 +10,8 @@ #include #include #include -#include #include #include -#include #include @@ -23,35 +21,28 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int FUNCTION_NOT_ALLOWED; } -template +template class FunctionsMultiStringFuzzySearch : public IFunction { - static_assert(LimitArgs > 0); - public: static constexpr auto name = Impl::name; + static FunctionPtr create(ContextPtr context) { - if (Impl::is_using_hyperscan && !context->getSettingsRef().allow_hyperscan) - throw Exception( - "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); - - return std::make_shared( - context->getSettingsRef().max_hyperscan_regexp_length, context->getSettingsRef().max_hyperscan_regexp_total_length); + const auto & settings = context->getSettingsRef(); + return std::make_shared(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } - FunctionsMultiStringFuzzySearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) - : max_hyperscan_regexp_length(max_hyperscan_regexp_length_), max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) - { - } + FunctionsMultiStringFuzzySearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) + : allow_hyperscan(allow_hyperscan_) + , max_hyperscan_regexp_length(max_hyperscan_regexp_length_) + , max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) + {} String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 3; } bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } @@ -60,82 +51,53 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); if (!isUnsignedInteger(arguments[1])) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName()); const DataTypeArray * array_type = checkAndGetDataType(arguments[2].get()); if (!array_type || !checkAndGetDataType(array_type->getNestedType().get())) - throw Exception( - "Illegal type " + arguments[2]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[2]->getName(), getName()); + return Impl::getReturnType(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - using ResultType = typename Impl::ResultType; - const ColumnPtr & column_haystack = arguments[0].column; + const ColumnPtr & num_ptr = arguments[1].column; + const ColumnPtr & arr_ptr = arguments[2].column; const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); + assert(col_haystack_vector); // getReturnTypeImpl() checks the data type - const ColumnPtr & num_ptr = arguments[1].column; - const ColumnConst * col_const_num = nullptr; UInt32 edit_distance = 0; - - if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) - edit_distance = col_const_num->getValue(); - else if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) - edit_distance = col_const_num->getValue(); - else if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) - edit_distance = col_const_num->getValue(); + if (const auto * col_const_uint8 = checkAndGetColumnConst(num_ptr.get())) + edit_distance = col_const_uint8->getValue(); + else if (const auto * col_const_uint16 = checkAndGetColumnConst(num_ptr.get())) + edit_distance = col_const_uint16->getValue(); + else if (const auto * col_const_uint32 = checkAndGetColumnConst(num_ptr.get())) + edit_distance = col_const_uint32->getValue(); else - throw Exception( - "Illegal column " + arguments[1].column->getName() - + ". The number is not const or does not fit in UInt32", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The number is not const or does not fit in UInt32", arguments[1].column->getName()); - - const ColumnPtr & arr_ptr = arguments[2].column; const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); - if (!col_const_arr) - throw Exception( - "Illegal column " + arguments[2].column->getName() + ". The array is not const", - ErrorCodes::ILLEGAL_COLUMN); - - Array src_arr = col_const_arr->getValue(); - - if (src_arr.size() > LimitArgs) - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + std::to_string(src_arr.size()) - + ", should be at most " + std::to_string(LimitArgs), - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - std::vector refs; - refs.reserve(src_arr.size()); - - for (const auto & el : src_arr) - refs.emplace_back(el.get()); - - if (Impl::is_using_hyperscan) - checkRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[2].column->getName()); + using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); + // the implementations are responsible for resizing the output column - /// The blame for resizing output is for the callee. - if (col_haystack_vector) - Impl::vectorConstant( - col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance); - else - throw Exception("Illegal column " + arguments[0].column->getName(), ErrorCodes::ILLEGAL_COLUMN); + Array needles_arr = col_const_arr->getValue(); + Impl::vectorConstant( + col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), needles_arr, vec_res, offsets_res, edit_distance, + allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); @@ -144,8 +106,9 @@ public: } private: - size_t max_hyperscan_regexp_length; - size_t max_hyperscan_regexp_total_length; + const bool allow_hyperscan; + const size_t max_hyperscan_regexp_length; + const size_t max_hyperscan_regexp_total_length; }; } diff --git a/src/Functions/FunctionsMultiStringPosition.h b/src/Functions/FunctionsMultiStringPosition.h index 357606f4042..855b5448b87 100644 --- a/src/Functions/FunctionsMultiStringPosition.h +++ b/src/Functions/FunctionsMultiStringPosition.h @@ -98,7 +98,8 @@ public: + ", should be at most 255", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - std::vector refs; + std::vector refs; + refs.reserve(src_arr.size()); for (const auto & el : src_arr) refs.emplace_back(el.get()); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 4bc8af3f214..04235e0a97a 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -10,15 +10,17 @@ #include #include #include -#include #include #include -#include namespace DB { /** + * multiMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) + * multiMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n]) + * multiMatchAllIndices(haystack, [pattern_1, pattern_2, ..., pattern_n]) + * * multiSearchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- find any of the const patterns inside haystack and return 0 or 1 * multiSearchAnyUTF8(haystack, [pattern_1, pattern_2, ..., pattern_n]) * multiSearchAnyCaseInsensitive(haystack, [pattern_1, pattern_2, ..., pattern_n]) @@ -34,37 +36,28 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int FUNCTION_NOT_ALLOWED; } -/// The argument limiting raises from Volnitsky searcher -- it is performance crucial to save only one byte for pattern number. -/// But some other searchers use this function, for example, multiMatchAny -- hyperscan does not have such restrictions -template ::max()> +template class FunctionsMultiStringSearch : public IFunction { - static_assert(LimitArgs > 0); - public: static constexpr auto name = Impl::name; + static FunctionPtr create(ContextPtr context) { - if (Impl::is_using_hyperscan && !context->getSettingsRef().allow_hyperscan) - throw Exception( - "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); - - return std::make_shared( - context->getSettingsRef().max_hyperscan_regexp_length, context->getSettingsRef().max_hyperscan_regexp_total_length); + const auto & settings = context->getSettingsRef(); + return std::make_shared(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } - FunctionsMultiStringSearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) - : max_hyperscan_regexp_length(max_hyperscan_regexp_length_), max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) - { - } + FunctionsMultiStringSearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) + : allow_hyperscan(allow_hyperscan_) + , max_hyperscan_regexp_length(max_hyperscan_regexp_length_) + , max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) + {} String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 2; } bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } @@ -73,60 +66,39 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); const DataTypeArray * array_type = checkAndGetDataType(arguments[1].get()); if (!array_type || !checkAndGetDataType(array_type->getNestedType().get())) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName()); + return Impl::getReturnType(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - using ResultType = typename Impl::ResultType; - const ColumnPtr & column_haystack = arguments[0].column; + const ColumnPtr & arr_ptr = arguments[1].column; const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); + assert(col_haystack_vector); // getReturnTypeImpl() checks the data type - const ColumnPtr & arr_ptr = arguments[1].column; const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); - if (!col_const_arr) - throw Exception( - "Illegal column " + arguments[1].column->getName() + ". The array is not const", - ErrorCodes::ILLEGAL_COLUMN); - - Array src_arr = col_const_arr->getValue(); - - if (src_arr.size() > LimitArgs) - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + std::to_string(src_arr.size()) - + ", should be at most " + std::to_string(LimitArgs), - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - std::vector refs; - refs.reserve(src_arr.size()); - - for (const auto & el : src_arr) - refs.emplace_back(el.get()); - - if (Impl::is_using_hyperscan) - checkRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[1].column->getName()); + using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); + // the implementations are responsible for resizing the output column - /// The blame for resizing output is for the callee. - if (col_haystack_vector) - Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res); - else - throw Exception("Illegal column " + arguments[0].column->getName(), ErrorCodes::ILLEGAL_COLUMN); + Array needles_arr = col_const_arr->getValue(); + Impl::vectorConstant( + col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), needles_arr, vec_res, offsets_res, + allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); @@ -135,8 +107,9 @@ public: } private: - size_t max_hyperscan_regexp_length; - size_t max_hyperscan_regexp_total_length; + const bool allow_hyperscan; + const size_t max_hyperscan_regexp_length; + const size_t max_hyperscan_regexp_total_length; }; } diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index adf9e9b585f..9c60dbffe91 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -1,15 +1,17 @@ #pragma once #include +#include #include #include #include +#include #include "Regexps.h" #include "config_functions.h" #include -#if USE_HYPERSCAN +#if USE_VECTORSCAN # include #endif @@ -19,18 +21,19 @@ namespace DB namespace ErrorCodes { - extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int CANNOT_ALLOCATE_MEMORY; + extern const int FUNCTION_NOT_ALLOWED; + extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int NOT_IMPLEMENTED; extern const int TOO_MANY_BYTES; } -template +template struct MultiMatchAllIndicesImpl { - using ResultType = Type; - static constexpr bool is_using_hyperscan = true; + using ResultType = ResultType_; + /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = true; @@ -44,24 +47,39 @@ struct MultiMatchAllIndicesImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, - PaddedPODArray & res, - PaddedPODArray & offsets) + const Array & needles_arr, + PaddedPODArray & res, + PaddedPODArray & offsets, + bool allow_hyperscan, + size_t max_hyperscan_regexp_length, + size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt); + vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( - const ColumnString::Chars & haystack_data, - const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, - PaddedPODArray & res, - PaddedPODArray & offsets, - [[maybe_unused]] std::optional edit_distance) + [[maybe_unused]] const ColumnString::Chars & haystack_data, + [[maybe_unused]] const ColumnString::Offsets & haystack_offsets, + [[maybe_unused]] const Array & needles_arr, + [[maybe_unused]] PaddedPODArray & res, + [[maybe_unused]] PaddedPODArray & offsets, + [[maybe_unused]] std::optional edit_distance, + bool allow_hyperscan, + [[maybe_unused]] size_t max_hyperscan_regexp_length, + [[maybe_unused]] size_t max_hyperscan_regexp_total_length) { + if (!allow_hyperscan) + throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); +#if USE_VECTORSCAN + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + offsets.resize(haystack_offsets.size()); -#if USE_HYPERSCAN - const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); + const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); @@ -76,7 +94,7 @@ struct MultiMatchAllIndicesImpl unsigned int /* flags */, void * context) -> int { - static_cast*>(context)->push_back(id); + static_cast*>(context)->push_back(id); return 0; }; const size_t haystack_offsets_size = haystack_offsets.size(); @@ -97,20 +115,15 @@ struct MultiMatchAllIndicesImpl on_match, &res); if (err != HS_SUCCESS) - throw Exception("Failed to scan with hyperscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); + throw Exception("Failed to scan with vectorscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); offsets[i] = res.size(); offset = haystack_offsets[i]; } #else - (void)haystack_data; - (void)haystack_offsets; - (void)needles; - (void)res; - (void)offsets; throw Exception( - "multi-search all indices is not implemented when hyperscan is off (is it x86 processor?)", + "multi-search all indices is not implemented when vectorscan is off", ErrorCodes::NOT_IMPLEMENTED); -#endif // USE_HYPERSCAN +#endif // USE_VECTORSCAN } }; diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 8a65c8cb2b4..0752e87e8af 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -1,14 +1,16 @@ #pragma once #include +#include #include #include +#include #include "Regexps.h" #include "config_functions.h" #include -#if USE_HYPERSCAN +#if USE_VECTORSCAN # include #else # include "MatchImpl.h" @@ -20,19 +22,31 @@ namespace DB namespace ErrorCodes { - extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int CANNOT_ALLOCATE_MEMORY; + extern const int FUNCTION_NOT_ALLOWED; + extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int NOT_IMPLEMENTED; extern const int TOO_MANY_BYTES; } +// For more readable instantiations of MultiMatchAnyImpl<> +struct MultiMatchTraits +{ +enum class Find +{ + Any, + AnyIndex +}; +}; -template +template struct MultiMatchAnyImpl { - static_assert(static_cast(FindAny) + static_cast(FindAnyIndex) == 1); - using ResultType = Type; - static constexpr bool is_using_hyperscan = true; + using ResultType = ResultType_; + + static constexpr bool FindAny = (Find == MultiMatchTraits::Find::Any); + static constexpr bool FindAnyIndex = (Find == MultiMatchTraits::Find::AnyIndex); + /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -46,31 +60,45 @@ struct MultiMatchAnyImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, - PaddedPODArray & res, - PaddedPODArray & offsets) + const Array & needles_arr, + PaddedPODArray & res, + PaddedPODArray & offsets, + bool allow_hyperscan, + size_t max_hyperscan_regexp_length, + size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt); + vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, - PaddedPODArray & res, + const Array & needles_arr, + PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, - [[maybe_unused]] std::optional edit_distance) + [[maybe_unused]] std::optional edit_distance, + bool allow_hyperscan, + size_t max_hyperscan_regexp_length, + size_t max_hyperscan_regexp_total_length) { - (void)FindAny; - (void)FindAnyIndex; + if (!allow_hyperscan) + throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + res.resize(haystack_offsets.size()); -#if USE_HYPERSCAN - const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); +#if USE_VECTORSCAN + const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); if (err != HS_SUCCESS) - throw Exception("Could not clone scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + throw Exception("Could not clone scratch space for vectorscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); MultiRegexps::ScratchPtr smart_scratch(scratch); @@ -81,9 +109,9 @@ struct MultiMatchAnyImpl void * context) -> int { if constexpr (FindAnyIndex) - *reinterpret_cast(context) = id; + *reinterpret_cast(context) = id; else if constexpr (FindAny) - *reinterpret_cast(context) = 1; + *reinterpret_cast(context) = 1; /// Once we hit the callback, there is no need to search for others. return 1; }; @@ -92,7 +120,7 @@ struct MultiMatchAnyImpl for (size_t i = 0; i < haystack_offsets_size; ++i) { UInt64 length = haystack_offsets[i] - offset - 1; - /// Hyperscan restriction. + /// Vectorscan restriction. if (length > std::numeric_limits::max()) throw Exception("Too long string to search", ErrorCodes::TOO_MANY_BYTES); /// Zero the result, scan, check, update the offset. @@ -106,21 +134,21 @@ struct MultiMatchAnyImpl on_match, &res[i]); if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) - throw Exception("Failed to scan with hyperscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); + throw Exception("Failed to scan with vectorscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); offset = haystack_offsets[i]; } #else - /// Fallback if do not use hyperscan - if constexpr (MultiSearchDistance) + // fallback if vectorscan is not compiled + if constexpr (WithEditDistance) throw Exception( - "Edit distance multi-search is not implemented when hyperscan is off (is it x86 processor?)", + "Edit distance multi-search is not implemented when vectorscan is off", ErrorCodes::NOT_IMPLEMENTED); PaddedPODArray accum(res.size()); memset(res.data(), 0, res.size() * sizeof(res.front())); memset(accum.data(), 0, accum.size()); for (size_t j = 0; j < needles.size(); ++j) { - MatchImpl::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum); + MatchImpl::vectorConstant(haystack_data, haystack_offsets, std::string(needles[j].data(), needles[j].size()), nullptr, accum); for (size_t i = 0; i < res.size(); ++i) { if constexpr (FindAny) @@ -129,7 +157,7 @@ struct MultiMatchAnyImpl res[i] = j + 1; } } -#endif // USE_HYPERSCAN +#endif // USE_VECTORSCAN } }; diff --git a/src/Functions/MultiSearchAllPositionsImpl.h b/src/Functions/MultiSearchAllPositionsImpl.h index f54fe41f20c..4356d6110f1 100644 --- a/src/Functions/MultiSearchAllPositionsImpl.h +++ b/src/Functions/MultiSearchAllPositionsImpl.h @@ -15,7 +15,7 @@ struct MultiSearchAllPositionsImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res) { auto res_callback = [](const UInt8 * start, const UInt8 * end) -> UInt64 diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index 26709119f6e..f69a3edbf8b 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -1,17 +1,22 @@ #pragma once #include +#include #include namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + template struct MultiSearchFirstIndexImpl { using ResultType = UInt64; - static constexpr bool is_using_hyperscan = false; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -22,10 +27,24 @@ struct MultiSearchFirstIndexImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, - [[maybe_unused]] PaddedPODArray & offsets) + [[maybe_unused]] PaddedPODArray & offsets, + bool /*allow_hyperscan*/, + size_t /*max_hyperscan_regexp_length*/, + size_t /*max_hyperscan_regexp_total_length*/) { + // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. + if (needles_arr.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + name, std::to_string(needles_arr.size()), std::to_string(std::numeric_limits::max())); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); res.resize(haystack_string_size); diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 1db8dcbde83..21d558a6d58 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -1,17 +1,22 @@ #pragma once #include +#include #include namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + template struct MultiSearchFirstPositionImpl { using ResultType = UInt64; - static constexpr bool is_using_hyperscan = false; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -22,10 +27,24 @@ struct MultiSearchFirstPositionImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, - [[maybe_unused]] PaddedPODArray & offsets) + [[maybe_unused]] PaddedPODArray & offsets, + bool /*allow_hyperscan*/, + size_t /*max_hyperscan_regexp_length*/, + size_t /*max_hyperscan_regexp_total_length*/) { + // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. + if (needles_arr.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + name, std::to_string(needles_arr.size()), std::to_string(std::numeric_limits::max())); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + auto res_callback = [](const UInt8 * start, const UInt8 * end) -> UInt64 { return 1 + Impl::countChars(reinterpret_cast(start), reinterpret_cast(end)); diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index 7cb0cefe580..1124184f58c 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -1,17 +1,22 @@ #pragma once #include +#include #include namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + template struct MultiSearchImpl { using ResultType = UInt8; - static constexpr bool is_using_hyperscan = false; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -22,10 +27,24 @@ struct MultiSearchImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, - [[maybe_unused]] PaddedPODArray & offsets) + [[maybe_unused]] PaddedPODArray & offsets, + bool /*allow_hyperscan*/, + size_t /*max_hyperscan_regexp_length*/, + size_t /*max_hyperscan_regexp_total_length*/) { + // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. + if (needles_arr.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + name, std::to_string(needles_arr.size()), std::to_string(std::numeric_limits::max())); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); res.resize(haystack_string_size); diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index 82e58cdc643..76f10373a58 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -26,7 +26,7 @@ struct PositionCaseSensitiveASCII using MultiSearcherInBigHaystack = MultiVolnitsky; /// For searching single substring, that is different each time. This object is created for each row of data. It must have cheap initialization. - using SearcherInSmallHaystack = LibCASCIICaseSensitiveStringSearcher; + using SearcherInSmallHaystack = StdLibASCIIStringSearcher; static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint) { @@ -38,7 +38,7 @@ struct PositionCaseSensitiveASCII return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } @@ -62,7 +62,7 @@ struct PositionCaseInsensitiveASCII /// `Volnitsky` is not used here, because one person has measured that this is better. It will be good if you question it. using SearcherInBigHaystack = ASCIICaseInsensitiveStringSearcher; using MultiSearcherInBigHaystack = MultiVolnitskyCaseInsensitive; - using SearcherInSmallHaystack = LibCASCIICaseInsensitiveStringSearcher; + using SearcherInSmallHaystack = StdLibASCIIStringSearcher; static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t /*haystack_size_hint*/) { @@ -74,7 +74,7 @@ struct PositionCaseInsensitiveASCII return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } @@ -94,7 +94,7 @@ struct PositionCaseSensitiveUTF8 { using SearcherInBigHaystack = VolnitskyUTF8; using MultiSearcherInBigHaystack = MultiVolnitskyUTF8; - using SearcherInSmallHaystack = LibCASCIICaseSensitiveStringSearcher; + using SearcherInSmallHaystack = StdLibASCIIStringSearcher; static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint) { @@ -106,7 +106,7 @@ struct PositionCaseSensitiveUTF8 return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } @@ -154,7 +154,7 @@ struct PositionCaseInsensitiveUTF8 return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 952e27b29bc..b932b14a6a9 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -16,7 +16,7 @@ #include "config_functions.h" -#if USE_HYPERSCAN +#if USE_VECTORSCAN # include #endif @@ -103,7 +103,7 @@ private: } -#if USE_HYPERSCAN +#if USE_VECTORSCAN namespace MultiRegexps { @@ -145,7 +145,7 @@ public: Regexps * operator()() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (regexp) return &*regexp; regexp = constructor(); @@ -166,10 +166,9 @@ struct Pool std::map, std::optional>, RegexpsConstructor> storage; }; -template -inline Regexps constructRegexps(const std::vector & str_patterns, std::optional edit_distance) +template +inline Regexps constructRegexps(const std::vector & str_patterns, [[maybe_unused]] std::optional edit_distance) { - (void)edit_distance; /// Common pointers std::vector patterns; std::vector flags; @@ -181,7 +180,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o patterns.reserve(str_patterns.size()); flags.reserve(str_patterns.size()); - if constexpr (CompileForEditDistance) + if constexpr (WithEditDistance) { ext_exprs.reserve(str_patterns.size()); ext_exprs_ptrs.reserve(str_patterns.size()); @@ -199,7 +198,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o * as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag */ flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8); - if constexpr (CompileForEditDistance) + if constexpr (WithEditDistance) { /// Hyperscan currently does not support UTF8 matching with edit distance. flags.back() &= ~HS_FLAG_UTF8; @@ -224,7 +223,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o } hs_error_t err; - if constexpr (!CompileForEditDistance) + if constexpr (!WithEditDistance) err = hs_compile_multi( patterns.data(), flags.data(), @@ -270,23 +269,22 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o if (err != HS_SUCCESS) throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); - return Regexps{db, scratch}; + return {db, scratch}; } -/// If CompileForEditDistance is False, edit_distance must be nullopt +/// If WithEditDistance is False, edit_distance must be nullopt /// Also, we use templates here because each instantiation of function /// template has its own copy of local static variables which must not be the same /// for different hyperscan compilations. -template -inline Regexps * get(const std::vector & patterns, std::optional edit_distance) +template +inline Regexps * get(const std::vector & patterns, std::optional edit_distance) { - /// C++11 has thread-safe function-local static on most modern compilers. - static Pool known_regexps; /// Different variables for different pattern parameters. + static Pool known_regexps; /// Different variables for different pattern parameters, thread-safe in C++11 std::vector str_patterns; str_patterns.reserve(patterns.size()); - for (const StringRef & ref : patterns) - str_patterns.push_back(ref.toString()); + for (const auto & pattern : patterns) + str_patterns.emplace_back(std::string(pattern.data(), pattern.size())); /// Get the lock for finding database. std::unique_lock lock(known_regexps.mutex); @@ -301,7 +299,7 @@ inline Regexps * get(const std::vector & patterns, std::optionalsecond.setConstructor([&str_patterns = it->first.first, edit_distance]() { - return constructRegexps(str_patterns, edit_distance); + return constructRegexps(str_patterns, edit_distance); }); } @@ -312,6 +310,6 @@ inline Regexps * get(const std::vector & patterns, std::optional +#include #include @@ -9,16 +9,16 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -void checkRegexp(const std::vector & refs, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) +void checkHyperscanRegexp(const std::vector & regexps, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { if (max_hyperscan_regexp_length > 0 || max_hyperscan_regexp_total_length > 0) { size_t total_regexp_length = 0; - for (const auto & pattern : refs) + for (const auto & regexp : regexps) { - if (max_hyperscan_regexp_length > 0 && pattern.size > max_hyperscan_regexp_length) + if (max_hyperscan_regexp_length > 0 && regexp.size() > max_hyperscan_regexp_length) throw Exception("Regexp length too large", ErrorCodes::BAD_ARGUMENTS); - total_regexp_length += pattern.size; + total_regexp_length += regexp.size(); } if (max_hyperscan_regexp_total_length > 0 && total_regexp_length > max_hyperscan_regexp_total_length) diff --git a/src/Functions/checkHyperscanRegexp.h b/src/Functions/checkHyperscanRegexp.h new file mode 100644 index 00000000000..2aac44115fc --- /dev/null +++ b/src/Functions/checkHyperscanRegexp.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace DB +{ + +void checkHyperscanRegexp(const std::vector & regexps, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length); + +} diff --git a/src/Functions/config_functions.h.in b/src/Functions/config_functions.h.in index a693611f975..04ae205aee2 100644 --- a/src/Functions/config_functions.h.in +++ b/src/Functions/config_functions.h.in @@ -2,6 +2,7 @@ // .h autogenerated by cmake! +#cmakedefine01 USE_BASEX #cmakedefine01 USE_BASE64 #cmakedefine01 USE_SIMDJSON #cmakedefine01 USE_RAPIDJSON @@ -9,4 +10,4 @@ #cmakedefine01 USE_S2_GEOMETRY #cmakedefine01 USE_FASTOPS #cmakedefine01 USE_NLP -#cmakedefine01 USE_HYPERSCAN +#cmakedefine01 USE_VECTORSCAN diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index 7615a2eeeaf..df2a8dc174c 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -1,6 +1,9 @@ if (TARGET ch_contrib::fastops) set(USE_FASTOPS 1) endif() +if (TARGET ch_contrib::base-x) + set(USE_BASEX 1) +endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() @@ -16,6 +19,6 @@ endif() if (TARGET ch_contrib::h3) set(USE_H3 1) endif() -if (TARGET ch_contrib::hyperscan) - set(USE_HYPERSCAN 1) +if (TARGET ch_contrib::vectorscan) + set(USE_VECTORSCAN 1) endif() diff --git a/src/Functions/filesystem.cpp b/src/Functions/filesystem.cpp index bea7ffdf818..36db68617e9 100644 --- a/src/Functions/filesystem.cpp +++ b/src/Functions/filesystem.cpp @@ -36,7 +36,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared>(std::filesystem::space(context->getConfigRef().getString("path"))); + return std::make_shared>(std::filesystem::space(context->getPath())); } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override diff --git a/src/Functions/hyperscanRegexpChecker.h b/src/Functions/hyperscanRegexpChecker.h deleted file mode 100644 index 1eea53722c1..00000000000 --- a/src/Functions/hyperscanRegexpChecker.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -void checkRegexp(const std::vector & refs, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length); - -} diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 95c66c20541..d4c2fcdd779 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -987,6 +987,8 @@ private: if (last_short_circuit_argument_index == -1) return; + executeColumnIfNeeded(arguments[0]); + /// Check if condition is const or null to not create full mask from it. if ((isColumnConst(*arguments[0].column) || arguments[0].column->onlyNull()) && !arguments[0].column->empty()) { diff --git a/src/Functions/multiFuzzyMatchAllIndices.cpp b/src/Functions/multiFuzzyMatchAllIndices.cpp index d0121ee3981..93ffb936dc1 100644 --- a/src/Functions/multiFuzzyMatchAllIndices.cpp +++ b/src/Functions/multiFuzzyMatchAllIndices.cpp @@ -13,9 +13,7 @@ struct NameMultiFuzzyMatchAllIndices static constexpr auto name = "multiFuzzyMatchAllIndices"; }; -using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch< - MultiMatchAllIndicesImpl, - std::numeric_limits::max()>; +using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiFuzzyMatchAny.cpp b/src/Functions/multiFuzzyMatchAny.cpp index 640e93a23b0..a627030d7af 100644 --- a/src/Functions/multiFuzzyMatchAny.cpp +++ b/src/Functions/multiFuzzyMatchAny.cpp @@ -13,9 +13,7 @@ struct NameMultiFuzzyMatchAny static constexpr auto name = "multiFuzzyMatchAny"; }; -using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiFuzzyMatchAnyIndex.cpp b/src/Functions/multiFuzzyMatchAnyIndex.cpp index f8bad1bc461..4b24a06e171 100644 --- a/src/Functions/multiFuzzyMatchAnyIndex.cpp +++ b/src/Functions/multiFuzzyMatchAnyIndex.cpp @@ -13,9 +13,7 @@ struct NameMultiFuzzyMatchAnyIndex static constexpr auto name = "multiFuzzyMatchAnyIndex"; }; -using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 7ed0ee00954..9e0ca4142e5 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -266,6 +266,8 @@ private: if (last_short_circuit_argument_index < 0) return; + executeColumnIfNeeded(arguments[0]); + /// Let's denote x_i' = maskedExecute(x_i, mask). /// multiIf(x_0, y_0, x_1, y_1, x_2, y_2, ..., x_{n-1}, y_{n-1}, y_n) /// We will support mask_i = !x_0 & !x_1 & ... & !x_i diff --git a/src/Functions/multiMatchAllIndices.cpp b/src/Functions/multiMatchAllIndices.cpp index 940c9e7e3bf..47bd57029e2 100644 --- a/src/Functions/multiMatchAllIndices.cpp +++ b/src/Functions/multiMatchAllIndices.cpp @@ -13,9 +13,7 @@ struct NameMultiMatchAllIndices static constexpr auto name = "multiMatchAllIndices"; }; -using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch< - MultiMatchAllIndicesImpl, - std::numeric_limits::max()>; +using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiMatchAny.cpp b/src/Functions/multiMatchAny.cpp index 47510e0ecc2..324e435de26 100644 --- a/src/Functions/multiMatchAny.cpp +++ b/src/Functions/multiMatchAny.cpp @@ -13,9 +13,7 @@ struct NameMultiMatchAny static constexpr auto name = "multiMatchAny"; }; -using FunctionMultiMatchAny = FunctionsMultiStringSearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiMatchAny = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiMatchAnyIndex.cpp b/src/Functions/multiMatchAnyIndex.cpp index a56d41dc95b..6a11fa4eb35 100644 --- a/src/Functions/multiMatchAnyIndex.cpp +++ b/src/Functions/multiMatchAnyIndex.cpp @@ -13,9 +13,7 @@ struct NameMultiMatchAnyIndex static constexpr auto name = "multiMatchAnyIndex"; }; -using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiSearchAnyCaseInsensitive.cpp b/src/Functions/multiSearchAnyCaseInsensitive.cpp index 9bc950c0d3d..af463805ea5 100644 --- a/src/Functions/multiSearchAnyCaseInsensitive.cpp +++ b/src/Functions/multiSearchAnyCaseInsensitive.cpp @@ -13,8 +13,7 @@ struct NameMultiSearchAnyCaseInsensitive { static constexpr auto name = "multiSearchAnyCaseInsensitive"; }; -using FunctionMultiSearchCaseInsensitive - = FunctionsMultiStringSearch>; +using FunctionMultiSearchCaseInsensitive = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiSearchFirstIndex.cpp b/src/Functions/multiSearchFirstIndex.cpp index a96ebed029c..a5fbec2dc36 100644 --- a/src/Functions/multiSearchFirstIndex.cpp +++ b/src/Functions/multiSearchFirstIndex.cpp @@ -14,8 +14,7 @@ struct NameMultiSearchFirstIndex static constexpr auto name = "multiSearchFirstIndex"; }; -using FunctionMultiSearchFirstIndex - = FunctionsMultiStringSearch>; +using FunctionMultiSearchFirstIndex = FunctionsMultiStringSearch>; } diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 9cd9c70da16..755d38409a6 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -40,6 +40,7 @@ void registerFunctionIsNaN(FunctionFactory &); void registerFunctionIfNotFinite(FunctionFactory &); void registerFunctionThrowIf(FunctionFactory &); void registerFunctionVersion(FunctionFactory &); +void registerFunctionRevision(FunctionFactory &); void registerFunctionBuildId(FunctionFactory &); void registerFunctionUptime(FunctionFactory &); void registerFunctionTimezone(FunctionFactory &); @@ -129,6 +130,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionIfNotFinite(factory); registerFunctionThrowIf(factory); registerFunctionVersion(factory); + registerFunctionRevision(factory); registerFunctionBuildId(factory); registerFunctionUptime(factory); registerFunctionTimezone(factory); diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index f86043c6959..248b6391b4f 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -49,6 +49,11 @@ void registerFunctionBase64Decode(FunctionFactory &); void registerFunctionTryBase64Decode(FunctionFactory &); #endif +#if USE_BASEX +void registerFunctionBase58Encode(FunctionFactory &); +void registerFunctionBase58Decode(FunctionFactory &); +#endif + #if USE_NLP void registerFunctionStem(FunctionFactory &); void registerFunctionSynonyms(FunctionFactory &); @@ -105,6 +110,11 @@ void registerFunctionsString(FunctionFactory & factory) registerFunctionTryBase64Decode(factory); #endif +#if USE_BASEX + registerFunctionBase58Encode(factory); + registerFunctionBase58Decode(factory); +#endif + #if USE_NLP registerFunctionStem(factory); registerFunctionSynonyms(factory); diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index 12134cf4e4c..e809ec7c298 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #if defined(OS_LINUX) # include @@ -88,6 +89,15 @@ namespace explicit FunctionVersion(ContextPtr context) : FunctionConstantBase(VERSION_STRING, context->isDistributed()) {} }; + /// revision() - returns the current revision. + class FunctionRevision : public FunctionConstantBase + { + public: + static constexpr auto name = "revision"; + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + explicit FunctionRevision(ContextPtr context) : FunctionConstantBase(ClickHouseRevision::getVersionRevision(), context->isDistributed()) {} + }; + class FunctionZooKeeperSessionUptime : public FunctionConstantBase { public: @@ -151,6 +161,11 @@ void registerFunctionVersion(FunctionFactory & factory) factory.registerFunction(FunctionFactory::CaseInsensitive); } +void registerFunctionRevision(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} + void registerFunctionZooKeeperSessionUptime(FunctionFactory & factory) { factory.registerFunction(); diff --git a/src/Functions/toColumnTypeName.cpp b/src/Functions/toColumnTypeName.cpp index 345de2954c4..466a8ba17c5 100644 --- a/src/Functions/toColumnTypeName.cpp +++ b/src/Functions/toColumnTypeName.cpp @@ -28,14 +28,6 @@ public: bool useDefaultImplementationForNothing() const override { return false; } - bool isShortCircuit(ShortCircuitSettings & settings, size_t /*number_of_arguments*/) const override - { - settings.enable_lazy_execution_for_first_argument = true; - settings.enable_lazy_execution_for_common_descendants_of_arguments = true; - settings.force_enable_lazy_execution = true; - return true; - } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } size_t getNumberOfArguments() const override diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index f3af49315ed..d9ec08642ca 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -32,13 +32,6 @@ public: bool useDefaultImplementationForNothing() const override { return false; } - bool isShortCircuit(ShortCircuitSettings & settings, size_t /*number_of_arguments*/) const override - { - settings.enable_lazy_execution_for_first_argument = false; - settings.enable_lazy_execution_for_common_descendants_of_arguments = true; - settings.force_enable_lazy_execution = true; - return true; - } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 0d571d6ae7c..479e0e88fcc 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -181,7 +181,7 @@ public: } private: - void nextImpl() override final + void nextImpl() final { const size_t prev_size = Base::position() - memory.data(); memory.resize(2 * prev_size + 1); diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 7ed2f343209..b31ee08eaf5 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -212,7 +212,7 @@ namespace size_t max_connections_per_endpoint, bool resolve_host = true) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); const std::string & host = uri.getHost(); UInt16 port = uri.getPort(); bool https = isHTTPS(uri); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 8fe8e2aa23e..4cd07dddf25 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -1065,6 +1065,8 @@ inline bool tryReadText(is_integer auto & x, ReadBuffer & buf) return tryReadIntText(x, buf); } +inline bool tryReadText(UUID & x, ReadBuffer & buf) { return tryReadUUIDText(x, buf); } + inline void readText(is_floating_point auto & x, ReadBuffer & buf) { readFloatText(x, buf); } inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); } diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index a58732287e9..cea03277c91 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -148,7 +148,7 @@ public: { String credentials_string; { - std::unique_lock locker(token_mutex); + std::lock_guard locker(token_mutex); LOG_TRACE(logger, "Getting default credentials for EC2 instance."); auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr); @@ -194,7 +194,7 @@ public: String new_token; { - std::unique_lock locker(token_mutex); + std::lock_guard locker(token_mutex); Aws::StringStream ss; ss << endpoint << EC2_IMDS_TOKEN_RESOURCE; diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index bd2a628355e..1fc21e1ac17 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -1,11 +1,10 @@ #pragma once #include -#include #include #include #include -#include +#include #include #include diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index d74b366b8e2..521acb6c8d6 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -64,7 +64,7 @@ public: } private: - void finalizeImpl() override final + void finalizeImpl() override { vector.resize( ((position() - reinterpret_cast(vector.data())) /// NOLINT diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index eb073ee8752..2703773f464 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1786,7 +1786,9 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(NodeRawConstPtrs conjunctio actions->inputs.push_back(input); } - actions->index.push_back(input); + /// We should not add result_predicate into the index for the second time. + if (input->result_name != result_predicate->result_name) + actions->index.push_back(input); } return actions; @@ -1840,13 +1842,14 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( if (can_remove_filter) { /// If filter column is not needed, remove it from index. - for (auto i = index.begin(); i != index.end(); ++i) + std::erase_if(index, [&](const Node * node) { return node == predicate; }); + + /// At the very end of this method we'll call removeUnusedActions() with allow_remove_inputs=false, + /// so we need to manually remove predicate if it is an input node. + if (predicate->type == ActionType::INPUT) { - if (*i == predicate) - { - index.erase(i); - break; - } + std::erase_if(inputs, [&](const Node * node) { return node == predicate; }); + nodes.remove_if([&](const Node & node) { return &node == predicate; }); } } else diff --git a/src/Interpreters/AggregateDescription.cpp b/src/Interpreters/AggregateDescription.cpp index 1a0748b5f97..b0f51ea7c90 100644 --- a/src/Interpreters/AggregateDescription.cpp +++ b/src/Interpreters/AggregateDescription.cpp @@ -82,24 +82,6 @@ void AggregateDescription::explain(WriteBuffer & out, size_t indent) const } out << "\n"; } - - out << prefix << " Argument positions: "; - - if (arguments.empty()) - out << "none\n"; - else - { - bool first = true; - for (auto arg : arguments) - { - if (!first) - out << ", "; - first = false; - - out << arg; - } - out << '\n'; - } } void AggregateDescription::explain(JSONBuilder::JSONMap & map) const @@ -137,15 +119,6 @@ void AggregateDescription::explain(JSONBuilder::JSONMap & map) const args_array->add(name); map.add("Arguments", std::move(args_array)); - - if (!arguments.empty()) - { - auto args_pos_array = std::make_unique(); - for (auto pos : arguments) - args_pos_array->add(pos); - - map.add("Argument Positions", std::move(args_pos_array)); - } } } diff --git a/src/Interpreters/AggregateDescription.h b/src/Interpreters/AggregateDescription.h index 12c14f7a57c..8c3302a8b0b 100644 --- a/src/Interpreters/AggregateDescription.h +++ b/src/Interpreters/AggregateDescription.h @@ -14,8 +14,7 @@ struct AggregateDescription { AggregateFunctionPtr function; Array parameters; /// Parameters of the (parametric) aggregate function. - ColumnNumbers arguments; - Names argument_names; /// used if no `arguments` are specified. + Names argument_names; String column_name; /// What name to use for a column with aggregate function values void explain(WriteBuffer & out, size_t indent) const; /// Get description for EXPLAIN query. @@ -23,5 +22,4 @@ struct AggregateDescription }; using AggregateDescriptions = std::vector; - } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 8a93dc5fd77..511e5c9e031 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -260,6 +260,14 @@ auto constructWithReserveIfPossible(size_t size_hint) else return std::make_unique(); } + +DB::ColumnNumbers calculateKeysPositions(const DB::Block & header, const DB::Aggregator::Params & params) +{ + DB::ColumnNumbers keys_positions(params.keys_size); + for (size_t i = 0; i < params.keys_size; ++i) + keys_positions[i] = header.getPositionByName(params.keys[i]); + return keys_positions; +} } namespace DB @@ -356,21 +364,17 @@ Aggregator::Params::StatsCollectingParams::StatsCollectingParams( Block Aggregator::getHeader(bool final) const { - return params.getHeader(final); + return params.getHeader(header, final); } Block Aggregator::Params::getHeader( - const Block & src_header, - const Block & intermediate_header, - const ColumnNumbers & keys, - const AggregateDescriptions & aggregates, - bool final) + const Block & header, bool only_merge, const Names & keys, const AggregateDescriptions & aggregates, bool final) { Block res; - if (intermediate_header) + if (only_merge) { - res = intermediate_header.cloneEmpty(); + res = header.cloneEmpty(); if (final) { @@ -386,14 +390,14 @@ Block Aggregator::Params::getHeader( else { for (const auto & key : keys) - res.insert(src_header.safeGetByPosition(key).cloneEmpty()); + res.insert(header.getByName(key).cloneEmpty()); for (const auto & aggregate : aggregates) { - size_t arguments_size = aggregate.arguments.size(); + size_t arguments_size = aggregate.argument_names.size(); DataTypes argument_types(arguments_size); for (size_t j = 0; j < arguments_size; ++j) - argument_types[j] = src_header.safeGetByPosition(aggregate.arguments[j]).type; + argument_types[j] = header.getByName(aggregate.argument_names[j]).type; DataTypePtr type; if (final) @@ -434,9 +438,6 @@ Aggregator::AggregateColumnsConstData Aggregator::Params::makeAggregateColumnsDa void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const { Strings res; - const auto & header = src_header ? src_header - : intermediate_header; - String prefix(indent, ' '); { @@ -444,16 +445,13 @@ void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const out << prefix << "Keys: "; bool first = true; - for (auto key : keys) + for (const auto & key : keys) { if (!first) out << ", "; first = false; - if (key >= header.columns()) - out << "unknown position " << key; - else - out << header.getByPosition(key).name; + out << key; } out << '\n'; @@ -470,18 +468,10 @@ void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const void Aggregator::Params::explain(JSONBuilder::JSONMap & map) const { - const auto & header = src_header ? src_header - : intermediate_header; - auto keys_array = std::make_unique(); - for (auto key : keys) - { - if (key >= header.columns()) - keys_array->add(""); - else - keys_array->add(header.getByPosition(key).name); - } + for (const auto & key : keys) + keys_array->add(key); map.add("Keys", std::move(keys_array)); @@ -526,7 +516,8 @@ public: #endif -Aggregator::Aggregator(const Params & params_) : params(params_) +Aggregator::Aggregator(const Block & header_, const Params & params_) + : header(header_), keys_positions(calculateKeysPositions(header, params_)), params(params_) { /// Use query-level memory tracker if (auto * memory_tracker_child = CurrentThread::getMemoryTracker()) @@ -672,9 +663,9 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() bool has_nullable_key = false; bool has_low_cardinality = false; - for (const auto & pos : params.keys) + for (const auto & key : params.keys) { - DataTypePtr type = (params.src_header ? params.src_header : params.intermediate_header).safeGetByPosition(pos).type; + DataTypePtr type = header.getByName(key).type; if (type->lowCardinality()) { @@ -1277,11 +1268,15 @@ void NO_INLINE Aggregator::mergeOnIntervalWithoutKeyImpl( } -void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns & aggregate_columns, Columns & materialized_columns, - AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) const +void Aggregator::prepareAggregateInstructions( + Columns columns, + AggregateColumns & aggregate_columns, + Columns & materialized_columns, + AggregateFunctionInstructions & aggregate_functions_instructions, + NestedColumnsHolder & nested_columns_holder) const { for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i].resize(params.aggregates[i].arguments.size()); + aggregate_columns[i].resize(params.aggregates[i].argument_names.size()); aggregate_functions_instructions.resize(params.aggregates_size + 1); aggregate_functions_instructions[params.aggregates_size].that = nullptr; @@ -1293,7 +1288,8 @@ void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns for (size_t j = 0; j < aggregate_columns[i].size(); ++j) { - materialized_columns.push_back(columns.at(params.aggregates[i].arguments[j])->convertToFullColumnIfConst()); + const auto pos = header.getPositionByName(params.aggregates[i].argument_names[j]); + materialized_columns.push_back(columns.at(pos)->convertToFullColumnIfConst()); aggregate_columns[i][j] = materialized_columns.back().get(); auto full_column = allow_sparse_arguments @@ -1382,7 +1378,7 @@ bool Aggregator::executeOnBlock(Columns columns, /// Remember the columns we will work with for (size_t i = 0; i < params.keys_size; ++i) { - materialized_columns.push_back(recursiveRemoveSparse(columns.at(params.keys[i]))->convertToFullColumnIfConst()); + materialized_columns.push_back(recursiveRemoveSparse(columns.at(keys_positions[i]))->convertToFullColumnIfConst()); key_columns[i] = materialized_columns.back().get(); if (!result.isLowCardinality()) @@ -1954,11 +1950,11 @@ Block Aggregator::prepareBlockAndFill( MutableColumns final_aggregate_columns(params.aggregates_size); AggregateColumnsData aggregate_columns_data(params.aggregates_size); - Block header = getHeader(final); + Block res_header = getHeader(final); for (size_t i = 0; i < params.keys_size; ++i) { - key_columns[i] = header.safeGetByPosition(i).type->createColumn(); + key_columns[i] = res_header.safeGetByPosition(i).type->createColumn(); key_columns[i]->reserve(rows); } @@ -1967,7 +1963,7 @@ Block Aggregator::prepareBlockAndFill( if (!final) { const auto & aggregate_column_name = params.aggregates[i].column_name; - aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); + aggregate_columns[i] = res_header.getByName(aggregate_column_name).type->createColumn(); /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. ColumnAggregateFunction & column_aggregate_func = assert_cast(*aggregate_columns[i]); @@ -2003,7 +1999,7 @@ Block Aggregator::prepareBlockAndFill( filler(key_columns, aggregate_columns_data, final_aggregate_columns, final); - Block res = header.cloneEmpty(); + Block res = res_header.cloneEmpty(); for (size_t i = 0; i < params.keys_size; ++i) res.getByPosition(i).column = std::move(key_columns[i]); @@ -2018,7 +2014,7 @@ Block Aggregator::prepareBlockAndFill( } /// Change the size of the columns-constants in the block. - size_t columns = header.columns(); + size_t columns = res_header.columns(); for (size_t i = 0; i < columns; ++i) if (isColumnConst(*res.getByPosition(i).column)) res.getByPosition(i).column = res.getByPosition(i).column->cut(0, rows); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 475fcd9e249..feb07727725 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -897,13 +897,8 @@ public: struct Params { - /// Data structure of source blocks. - Block src_header; - /// Data structure of intermediate blocks before merge. - Block intermediate_header; - /// What to count. - const ColumnNumbers keys; + const Names keys; const AggregateDescriptions aggregates; const size_t keys_size; const size_t aggregates_size; @@ -937,6 +932,8 @@ public: bool compile_aggregate_expressions; size_t min_count_to_compile_aggregate_expression; + bool only_merge; + struct StatsCollectingParams { StatsCollectingParams(); @@ -957,8 +954,7 @@ public: StatsCollectingParams stats_collecting_params; Params( - const Block & src_header_, - const ColumnNumbers & keys_, + const Names & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_rows_to_group_by_, @@ -972,11 +968,9 @@ public: size_t min_free_disk_space_, bool compile_aggregate_expressions_, size_t min_count_to_compile_aggregate_expression_, - const Block & intermediate_header_ = {}, + bool only_merge_ = false, // true for projections const StatsCollectingParams & stats_collecting_params_ = {}) - : src_header(src_header_) - , intermediate_header(intermediate_header_) - , keys(keys_) + : keys(keys_) , aggregates(aggregates_) , keys_size(keys.size()) , aggregates_size(aggregates.size()) @@ -992,33 +986,22 @@ public: , min_free_disk_space(min_free_disk_space_) , compile_aggregate_expressions(compile_aggregate_expressions_) , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) + , only_merge(only_merge_) , stats_collecting_params(stats_collecting_params_) { } /// Only parameters that matter during merge. - Params( - const Block & intermediate_header_, - const ColumnNumbers & keys_, - const AggregateDescriptions & aggregates_, - bool overflow_row_, - size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, {}, {}) + Params(const Names & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) + : Params( + keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, true, {}) { - intermediate_header = intermediate_header_; } - static Block getHeader( - const Block & src_header, - const Block & intermediate_header, - const ColumnNumbers & keys, - const AggregateDescriptions & aggregates, - bool final); + static Block + getHeader(const Block & header, bool only_merge, const Names & keys, const AggregateDescriptions & aggregates, bool final); - Block getHeader(bool final) const - { - return getHeader(src_header, intermediate_header, keys, aggregates, final); - } + Block getHeader(const Block & header_, bool final) const { return getHeader(header_, only_merge, keys, aggregates, final); } /// Remember the columns we will work with ColumnRawPtrs makeRawKeyColumns(const Block & block) const; @@ -1029,7 +1012,7 @@ public: void explain(JSONBuilder::JSONMap & map) const; }; - explicit Aggregator(const Params & params_); + explicit Aggregator(const Block & header_, const Params & params_); /// Process one block. Return false if the processing should be aborted (with group_by_overflow_mode = 'break'). bool executeOnBlock(const Block & block, @@ -1106,6 +1089,10 @@ private: friend class ConvertingAggregatedToChunksSource; friend class AggregatingInOrderTransform; + /// Data structure of source blocks. + Block header; + /// Positions of aggregation key columns in the header. + const ColumnNumbers keys_positions; Params params; AggregatedDataVariants::Type method_chosen; diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index fd8f252d139..ae39c7035dd 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -215,7 +215,7 @@ void AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) } } - std::unique_lock write_lock(rwlock); + std::lock_guard write_lock(rwlock); auto it = queue.emplace(key, std::make_shared()).first; pushImpl(std::move(entry), it); } @@ -343,7 +343,7 @@ void AsynchronousInsertQueue::cleanup() if (!keys_to_remove.empty()) { - std::unique_lock write_lock(rwlock); + std::lock_guard write_lock(rwlock); size_t total_removed = 0; for (const auto & key : keys_to_remove) diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index db3cb3049fd..8a4e8dad8dd 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include diff --git a/src/Interpreters/AsynchronousMetricLog.h b/src/Interpreters/AsynchronousMetricLog.h index 836e967a7a6..900d84868bd 100644 --- a/src/Interpreters/AsynchronousMetricLog.h +++ b/src/Interpreters/AsynchronousMetricLog.h @@ -40,7 +40,7 @@ struct AsynchronousMetricLogElement return "event_date Date CODEC(Delta(2), ZSTD(1)), " "event_time DateTime CODEC(Delta(4), ZSTD(1)), " "metric LowCardinality(String) CODEC(ZSTD(1)), " - "value Float64 CODEC(Gorilla, ZSTD(3))"; + "value Float64 CODEC(ZSTD(3))"; } }; diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 4ac5acfd60f..37ed418ec2a 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -615,8 +615,8 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti auto caches = FileCacheFactory::instance().getAll(); for (const auto & [_, cache_data] : caches) { - new_values["FilesystemCacheBytes"] = cache_data.cache->getUsedCacheSize(); - new_values["FilesystemCacheFiles"] = cache_data.cache->getFileSegmentsNum(); + new_values["FilesystemCacheBytes"] = cache_data->cache->getUsedCacheSize(); + new_values["FilesystemCacheFiles"] = cache_data->cache->getFileSegmentsNum(); } } diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index dc93b602a29..5dc6d4acbe5 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -124,7 +124,7 @@ protected: std::string queue_dir; /// dir with queue of queries mutable std::mutex zookeeper_mutex; - ZooKeeperPtr current_zookeeper; + ZooKeeperPtr current_zookeeper TSA_GUARDED_BY(zookeeper_mutex); /// Save state of executed task to avoid duplicate execution on ZK error std::optional last_skipped_entry_name; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index fa9444a7e66..bae2aed2cd5 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -16,8 +16,12 @@ #include #include #include -#include #include +#include + +#include +#include +#include #include "config_core.h" @@ -27,12 +31,10 @@ #endif #if USE_LIBPQXX -# include # include +# include #endif -namespace fs = std::filesystem; - namespace CurrentMetrics { extern const Metric TablesToDropQueueSize; @@ -77,6 +79,7 @@ TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryT } auto table_id = StorageID(DatabaseCatalog::TEMPORARY_DATABASE, global_name, id); auto table = creator(table_id); + DatabaseCatalog::instance().addUUIDMapping(id); temporary_tables->createTable(getContext(), global_name, table, original_create); table->startup(); } @@ -143,6 +146,9 @@ StoragePtr TemporaryTableHolder::getTable() const void DatabaseCatalog::initializeAndLoadTemporaryDatabase() { drop_delay_sec = getContext()->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec); + unused_dir_hide_timeout_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_hide_timeout_sec", unused_dir_hide_timeout_sec); + unused_dir_rm_timeout_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_rm_timeout_sec", unused_dir_rm_timeout_sec); + unused_dir_cleanup_period_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_cleanup_period_sec", unused_dir_cleanup_period_sec); auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE, getContext()); attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); @@ -150,6 +156,16 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::loadDatabases() { + if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && unused_dir_cleanup_period_sec) + { + auto cleanup_task_holder + = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this]() { this->cleanupStoreDirectoryTask(); }); + cleanup_task = std::make_unique(std::move(cleanup_task_holder)); + (*cleanup_task)->activate(); + /// Do not start task immediately on server startup, it's not urgent. + (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + } + auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); drop_task = std::make_unique(std::move(task_holder)); (*drop_task)->activate(); @@ -166,6 +182,9 @@ void DatabaseCatalog::shutdownImpl() { TemporaryLiveViewCleaner::shutdown(); + if (cleanup_task) + (*cleanup_task)->deactivate(); + if (drop_task) (*drop_task)->deactivate(); @@ -186,31 +205,42 @@ void DatabaseCatalog::shutdownImpl() for (auto & database : current_databases) database.second->shutdown(); - tables_marked_dropped.clear(); + { + std::lock_guard lock(tables_marked_dropped_mutex); + tables_marked_dropped.clear(); + } std::lock_guard lock(databases_mutex); + for (const auto & db : databases) + { + UUID db_uuid = db.second->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMapping(db_uuid); + } assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { /// Ensure that all UUID mappings are empty (i.e. all mappings contain nullptr instead of a pointer to storage) const auto & not_empty_mapping = [] (const auto & mapping) { + auto & db = mapping.second.first; auto & table = mapping.second.second; - return table; + return db || table; }; + std::lock_guard map_lock{elem.mutex}; auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); return it != elem.map.end(); }) == uuid_map.end()); databases.clear(); - db_uuid_map.clear(); view_dependencies.clear(); } -bool DatabaseCatalog::isPredefinedDatabaseName(const std::string_view & database_name) +bool DatabaseCatalog::isPredefinedDatabase(const std::string_view & database_name) { return database_name == TEMPORARY_DATABASE || database_name == SYSTEM_DATABASE || database_name == INFORMATION_SCHEMA || database_name == INFORMATION_SCHEMA_UPPERCASE; } + DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); @@ -299,6 +329,48 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( return {database, table}; } +bool DatabaseCatalog::isPredefinedTable(const StorageID & table_id) const +{ + static const char * information_schema_views[] = {"schemata", "tables", "views", "columns"}; + static const char * information_schema_views_uppercase[] = {"SCHEMATA", "TABLES", "VIEWS", "COLUMNS"}; + + auto check_database_and_table_name = [&](const String & database_name, const String & table_name) + { + if (database_name == SYSTEM_DATABASE) + { + auto storage = getSystemDatabase()->tryGetTable(table_name, getContext()); + return storage && storage->isSystemStorage(); + } + if (database_name == INFORMATION_SCHEMA) + { + return std::find(std::begin(information_schema_views), std::end(information_schema_views), table_name) + != std::end(information_schema_views); + } + if (database_name == INFORMATION_SCHEMA_UPPERCASE) + { + return std::find(std::begin(information_schema_views_uppercase), std::end(information_schema_views_uppercase), table_name) + != std::end(information_schema_views_uppercase); + } + return false; + }; + + if (table_id.hasUUID()) + { + if (auto storage = tryGetByUUID(table_id.uuid).second) + { + if (storage->isSystemStorage()) + return true; + auto res_id = storage->getStorageID(); + String database_name = res_id.getDatabaseName(); + if (database_name != SYSTEM_DATABASE) /// If (database_name == SYSTEM_DATABASE) then we have already checked it (see isSystemStorage() above). + return check_database_and_table_name(database_name, res_id.getTableName()); + } + return false; + } + + return check_database_and_table_name(table_id.getDatabaseName(), table_id.getTableName()); +} + void DatabaseCatalog::assertDatabaseExists(const String & database_name) const { std::lock_guard lock{databases_mutex}; @@ -331,9 +403,10 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases.emplace(database_name, database); + NOEXCEPT_SCOPE; UUID db_uuid = database->getUUID(); if (db_uuid != UUIDHelpers::Nil) - db_uuid_map.emplace(db_uuid, database); + addUUIDMapping(db_uuid, database, nullptr); } @@ -347,7 +420,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); db = databases.find(database_name)->second; - db_uuid_map.erase(db->getUUID()); + UUID db_uuid = db->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMapping(db_uuid); databases.erase(database_name); } @@ -372,6 +447,8 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (drop) { + UUID db_uuid = db->getUUID(); + /// Delete the database. db->drop(local_context); @@ -381,6 +458,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri fs::remove(database_metadata_dir); fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql"); fs::remove(database_metadata_file); + + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMappingFinally(db_uuid); } return db; @@ -427,21 +507,19 @@ DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const DatabasePtr DatabaseCatalog::getDatabase(const UUID & uuid) const { - std::lock_guard lock{databases_mutex}; - auto it = db_uuid_map.find(uuid); - if (it == db_uuid_map.end()) + auto db_and_table = tryGetByUUID(uuid); + if (!db_and_table.first || db_and_table.second) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database UUID {} does not exist", toString(uuid)); - return it->second; + return db_and_table.first; } DatabasePtr DatabaseCatalog::tryGetDatabase(const UUID & uuid) const { assert(uuid != UUIDHelpers::Nil); - std::lock_guard lock{databases_mutex}; - auto it = db_uuid_map.find(uuid); - if (it == db_uuid_map.end()) + auto db_and_table = tryGetByUUID(uuid); + if (!db_and_table.first || db_and_table.second) return {}; - return it->second; + return db_and_table.first; } bool DatabaseCatalog::isDatabaseExist(const String & database_name) const @@ -496,18 +574,22 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid) void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); - assert((database && table) || (!database && !table)); + assert(database || !table); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; auto [it, inserted] = map_part.map.try_emplace(uuid, database, table); if (inserted) + { + /// Mapping must be locked before actually inserting something + chassert((!database && !table)); return; + } auto & prev_database = it->second.first; auto & prev_table = it->second.second; - assert((prev_database && prev_table) || (!prev_database && !prev_table)); + assert(prev_database || !prev_table); - if (!prev_table && table) + if (!prev_database && database) { /// It's empty mapping, it was created to "lock" UUID and prevent collision. Just update it. prev_database = database; @@ -515,8 +597,8 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & data return; } - /// We are trying to replace existing mapping (prev_table != nullptr), it's logical error - if (table) + /// We are trying to replace existing mapping (prev_database != nullptr), it's logical error + if (database || table) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", toString(uuid)); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR @@ -560,6 +642,14 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, prev_table = std::move(table); } +bool DatabaseCatalog::hasUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + return map_part.map.contains(uuid); +} + std::unique_ptr DatabaseCatalog::database_catalog; DatabaseCatalog::DatabaseCatalog(ContextMutablePtr global_context_) @@ -646,7 +736,8 @@ DatabaseCatalog::updateDependency(const StorageID & old_from, const StorageID & DDLGuardPtr DatabaseCatalog::getDDLGuard(const String & database, const String & table) { std::unique_lock lock(ddl_guards_mutex); - auto db_guard_iter = ddl_guards.try_emplace(database).first; + /// TSA does not support unique_lock + auto db_guard_iter = TSA_SUPPRESS_WARNING_FOR_WRITE(ddl_guards).try_emplace(database).first; DatabaseGuard & db_guard = db_guard_iter->second; return std::make_unique(db_guard.first, db_guard.second, std::move(lock), table, database); } @@ -655,7 +746,7 @@ std::unique_lock DatabaseCatalog::getExclusiveDDLGuardForData { DDLGuards::iterator db_guard_iter; { - std::unique_lock lock(ddl_guards_mutex); + std::lock_guard lock(ddl_guards_mutex); db_guard_iter = ddl_guards.try_emplace(database).first; assert(db_guard_iter->second.first.contains("")); } @@ -701,6 +792,8 @@ DatabaseAndTable DatabaseCatalog::tryGetDatabaseAndTable(const StorageID & table void DatabaseCatalog::loadMarkedAsDroppedTables() { + assert(!cleanup_task); + /// /clickhouse_root/metadata_dropped/ contains files with metadata of tables, /// which where marked as dropped by Atomic databases. /// Data directories of such tables still exists in store/ @@ -780,6 +873,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr time_t drop_time; if (table) { + chassert(hasUUIDMapping(table_id.uuid)); drop_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); table->is_dropped = true; } @@ -953,7 +1047,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) LOG_DEBUG(log, "Waiting for table {} to be finally dropped", toString(uuid)); std::unique_lock lock{tables_marked_dropped_mutex}; - wait_table_finally_dropped.wait(lock, [&]() + wait_table_finally_dropped.wait(lock, [&]() TSA_REQUIRES(tables_marked_dropped_mutex) -> bool { return !tables_marked_dropped_ids.contains(uuid); }); @@ -1072,6 +1166,166 @@ void DatabaseCatalog::updateLoadingDependencies(const StorageID & table_id, Tabl old_dependencies = std::move(new_dependencies); } +void DatabaseCatalog::cleanupStoreDirectoryTask() +{ + fs::path store_path = fs::path(getContext()->getPath()) / "store"; + size_t affected_dirs = 0; + for (const auto & prefix_dir : fs::directory_iterator{store_path}) + { + String prefix = prefix_dir.path().filename(); + bool expected_prefix_dir = prefix_dir.is_directory() && + prefix.size() == 3 && + isHexDigit(prefix[0]) && + isHexDigit(prefix[1]) && + isHexDigit(prefix[2]); + + if (!expected_prefix_dir) + { + LOG_WARNING(log, "Found invalid directory {}, will try to remove it", prefix_dir.path().string()); + affected_dirs += maybeRemoveDirectory(prefix_dir.path()); + continue; + } + + for (const auto & uuid_dir : fs::directory_iterator{prefix_dir.path()}) + { + String uuid_str = uuid_dir.path().filename(); + UUID uuid; + bool parsed = tryParse(uuid, uuid_str); + + bool expected_dir = uuid_dir.is_directory() && + parsed && + uuid != UUIDHelpers::Nil && + uuid_str.starts_with(prefix); + + if (!expected_dir) + { + LOG_WARNING(log, "Found invalid directory {}, will try to remove it", uuid_dir.path().string()); + affected_dirs += maybeRemoveDirectory(uuid_dir.path()); + continue; + } + + /// Order is important + if (!hasUUIDMapping(uuid)) + { + /// We load uuids even for detached and permanently detached tables, + /// so it looks safe enough to remove directory if we don't have uuid mapping for it. + /// No table or database using this directory should concurrently appear, + /// because creation of new table would fail with "directory already exists". + affected_dirs += maybeRemoveDirectory(uuid_dir.path()); + } + } + } + + if (affected_dirs) + LOG_INFO(log, "Cleaned up {} directories from store/", affected_dirs); + + (*cleanup_task)->scheduleAfter(unused_dir_cleanup_period_sec * 1000); +} + +bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) +{ + /// "Safe" automatic removal of some directory. + /// At first we do not remove anything and only revoke all access right. + /// And remove only if nobody noticed it after, for example, one month. + + struct stat st; + if (stat(unused_dir.string().c_str(), &st)) + { + LOG_ERROR(log, "Failed to stat {}, errno: {}", unused_dir.string(), errno); + return false; + } + + if (st.st_uid != geteuid()) + { + /// Directory is not owned by clickhouse, it's weird, let's ignore it (chmod will likely fail anyway). + LOG_WARNING(log, "Found directory {} with unexpected owner (uid={})", unused_dir.string(), st.st_uid); + return false; + } + + time_t max_modification_time = std::max(st.st_atime, std::max(st.st_mtime, st.st_ctime)); + time_t current_time = time(nullptr); + if (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) + { + if (current_time <= max_modification_time + unused_dir_hide_timeout_sec) + return false; + + LOG_INFO(log, "Removing access rights for unused directory {} (will remove it when timeout exceed)", unused_dir.string()); + + /// Explicitly update modification time just in case + + struct utimbuf tb; + tb.actime = current_time; + tb.modtime = current_time; + if (utime(unused_dir.string().c_str(), &tb) != 0) + LOG_ERROR(log, "Failed to utime {}, errno: {}", unused_dir.string(), errno); + + /// Remove all access right + if (chmod(unused_dir.string().c_str(), 0)) + LOG_ERROR(log, "Failed to chmod {}, errno: {}", unused_dir.string(), errno); + + return true; + } + else + { + if (!unused_dir_rm_timeout_sec) + return false; + + if (current_time <= max_modification_time + unused_dir_rm_timeout_sec) + return false; + + LOG_INFO(log, "Removing unused directory {}", unused_dir.string()); + + /// We have to set these access rights to make recursive removal work + if (chmod(unused_dir.string().c_str(), S_IRWXU)) + LOG_ERROR(log, "Failed to chmod {}, errno: {}", unused_dir.string(), errno); + + fs::remove_all(unused_dir); + + return true; + } +} + +static void maybeUnlockUUID(UUID uuid) +{ + if (uuid == UUIDHelpers::Nil) + return; + + chassert(DatabaseCatalog::instance().hasUUIDMapping(uuid)); + auto db_and_table = DatabaseCatalog::instance().tryGetByUUID(uuid); + if (!db_and_table.first && !db_and_table.second) + { + DatabaseCatalog::instance().removeUUIDMappingFinally(uuid); + return; + } + chassert(db_and_table.first || !db_and_table.second); +} + +TemporaryLockForUUIDDirectory::TemporaryLockForUUIDDirectory(UUID uuid_) + : uuid(uuid_) +{ + if (uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().addUUIDMapping(uuid); +} + +TemporaryLockForUUIDDirectory::~TemporaryLockForUUIDDirectory() +{ + maybeUnlockUUID(uuid); +} + +TemporaryLockForUUIDDirectory::TemporaryLockForUUIDDirectory(TemporaryLockForUUIDDirectory && rhs) noexcept + : uuid(rhs.uuid) +{ + rhs.uuid = UUIDHelpers::Nil; +} + +TemporaryLockForUUIDDirectory & TemporaryLockForUUIDDirectory::operator = (TemporaryLockForUUIDDirectory && rhs) noexcept +{ + maybeUnlockUUID(uuid); + uuid = rhs.uuid; + rhs.uuid = UUIDHelpers::Nil; + return *this; +} + DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 79ba4052038..133cf0c5126 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -20,7 +20,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -128,8 +130,8 @@ public: static constexpr const char * INFORMATION_SCHEMA = "information_schema"; static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA"; - /// Returns true if a passed string is one of the predefined databases' names - static bool isPredefinedDatabaseName(const std::string_view & database_name); + /// Returns true if a passed name is one of the predefined databases' names. + static bool isPredefinedDatabase(const std::string_view & database_name); static DatabaseCatalog & init(ContextMutablePtr global_context_); static DatabaseCatalog & instance(); @@ -179,6 +181,11 @@ public: ContextPtr context, std::optional * exception = nullptr) const; + /// Returns true if a passed table_id refers to one of the predefined tables' names. + /// All tables in the "system" database with System* table engine are predefined. + /// Four views (tables, views, columns, schemata) in the "information_schema" database are predefined too. + bool isPredefinedTable(const StorageID & table_id) const; + void addDependency(const StorageID & from, const StorageID & where); void removeDependency(const StorageID & from, const StorageID & where); Dependencies getDependencies(const StorageID & from) const; @@ -203,6 +210,8 @@ public: /// this method will throw an exception. void addUUIDMapping(const UUID & uuid); + bool hasUUIDMapping(const UUID & uuid); + static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -217,7 +226,7 @@ public: DependenciesInfo getLoadingDependenciesInfo(const StorageID & table_id) const; TableNamesSet tryRemoveLoadingDependencies(const StorageID & table_id, bool check_dependencies, bool is_drop_database = false); - TableNamesSet tryRemoveLoadingDependenciesUnlocked(const QualifiedTableName & removing_table, bool check_dependencies, bool is_drop_database = false); + TableNamesSet tryRemoveLoadingDependenciesUnlocked(const QualifiedTableName & removing_table, bool check_dependencies, bool is_drop_database = false) TSA_REQUIRES(databases_mutex); void checkTableCanBeRemovedOrRenamed(const StorageID & table_id) const; void updateLoadingDependencies(const StorageID & table_id, TableNamesSet && new_dependencies); @@ -229,15 +238,15 @@ private: static std::unique_ptr database_catalog; explicit DatabaseCatalog(ContextMutablePtr global_context_); - void assertDatabaseExistsUnlocked(const String & database_name) const; - void assertDatabaseDoesntExistUnlocked(const String & database_name) const; + void assertDatabaseExistsUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex); + void assertDatabaseDoesntExistUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex); void shutdownImpl(); struct UUIDToStorageMapPart { - std::unordered_map map; + std::unordered_map map TSA_GUARDED_BY(mutex); mutable std::mutex mutex; }; @@ -261,20 +270,20 @@ private: void dropTableDataTask(); void dropTableFinally(const TableMarkedAsDropped & table); + void cleanupStoreDirectoryTask(); + bool maybeRemoveDirectory(const fs::path & unused_dir); + static constexpr size_t reschedule_time_ms = 100; static constexpr time_t drop_error_cooldown_sec = 5; - using UUIDToDatabaseMap = std::unordered_map; - mutable std::mutex databases_mutex; - ViewDependencies view_dependencies; + ViewDependencies view_dependencies TSA_GUARDED_BY(databases_mutex); - Databases databases; - UUIDToDatabaseMap db_uuid_map; + Databases databases TSA_GUARDED_BY(databases_mutex); UUIDToStorageMap uuid_map; - DependenciesInfos loading_dependencies; + DependenciesInfos loading_dependencies TSA_GUARDED_BY(databases_mutex); Poco::Logger * log; @@ -286,18 +295,45 @@ private: /// In case the element already exists, waits when query will be executed in other thread. See class DDLGuard below. using DatabaseGuard = std::pair; using DDLGuards = std::map; - DDLGuards ddl_guards; + DDLGuards ddl_guards TSA_GUARDED_BY(ddl_guards_mutex); /// If you capture mutex and ddl_guards_mutex, then you need to grab them strictly in this order. mutable std::mutex ddl_guards_mutex; - TablesMarkedAsDropped tables_marked_dropped; - std::unordered_set tables_marked_dropped_ids; + TablesMarkedAsDropped tables_marked_dropped TSA_GUARDED_BY(tables_marked_dropped_mutex); + std::unordered_set tables_marked_dropped_ids TSA_GUARDED_BY(tables_marked_dropped_mutex); mutable std::mutex tables_marked_dropped_mutex; std::unique_ptr drop_task; static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; std::condition_variable wait_table_finally_dropped; + + std::unique_ptr cleanup_task; + static constexpr time_t default_unused_dir_hide_timeout_sec = 60 * 60; /// 1 hour + time_t unused_dir_hide_timeout_sec = default_unused_dir_hide_timeout_sec; + static constexpr time_t default_unused_dir_rm_timeout_sec = 30 * 24 * 60 * 60; /// 30 days + time_t unused_dir_rm_timeout_sec = default_unused_dir_rm_timeout_sec; + static constexpr time_t default_unused_dir_cleanup_period_sec = 24 * 60 * 60; /// 1 day + time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; +}; + +/// This class is useful when creating a table or database. +/// Usually we create IStorage/IDatabase object first and then add it to IDatabase/DatabaseCatalog. +/// But such object may start using a directory in store/ since its creation. +/// To avoid race with cleanupStoreDirectoryTask() we have to mark UUID as used first. +/// Then we can either add DatabasePtr/StoragePtr to the created UUID mapping +/// or remove the lock if creation failed. +/// See also addUUIDMapping(...) +class TemporaryLockForUUIDDirectory : private boost::noncopyable +{ + UUID uuid = UUIDHelpers::Nil; +public: + TemporaryLockForUUIDDirectory() = default; + TemporaryLockForUUIDDirectory(UUID uuid_); + ~TemporaryLockForUUIDDirectory(); + + TemporaryLockForUUIDDirectory(TemporaryLockForUUIDDirectory && rhs) noexcept; + TemporaryLockForUUIDDirectory & operator = (TemporaryLockForUUIDDirectory && rhs) noexcept; }; } diff --git a/src/Interpreters/EmbeddedDictionaries.cpp b/src/Interpreters/EmbeddedDictionaries.cpp index b9bc5cd1e5c..0b2efaf3dbe 100644 --- a/src/Interpreters/EmbeddedDictionaries.cpp +++ b/src/Interpreters/EmbeddedDictionaries.cpp @@ -60,7 +60,7 @@ bool EmbeddedDictionaries::reloadDictionary( bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool force_reload) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); /** If you can not update the directories, then despite this, do not throw an exception (use the old directories). * If there are no old correct directories, then when using functions that depend on them, diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index b2a7aedaa5a..17d09b25ea1 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include @@ -339,7 +339,7 @@ public: /// We keep correspondence between used_flags and hash table internal buffer. /// Hash table cannot be modified during HashJoin lifetime and must be protected with lock. - void setLock(RWLockImpl::LockHolder rwlock_holder) + void setLock(TableLockHolder rwlock_holder) { storage_join_lock = rwlock_holder; } @@ -394,7 +394,7 @@ private: /// Should be set via setLock to protect hash table from modification from StorageJoin /// If set HashJoin instance is not available for modification (addJoinedBlock) - RWLockImpl::LockHolder storage_join_lock = nullptr; + TableLockHolder storage_join_lock = nullptr; void dataMapInit(MapsVariant &); diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index 6176e3cc865..2172a6f4261 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -38,7 +38,6 @@ MutableColumns InternalTextLogsQueue::getSampleColumns() void InternalTextLogsQueue::pushBlock(Block && log_block) { - OvercommitTrackerBlockerInThread blocker; static Block sample_block = getSampleBlock(); if (blocksHaveEqualStructure(sample_block, log_block)) diff --git a/src/Interpreters/InternalTextLogsQueue.h b/src/Interpreters/InternalTextLogsQueue.h index a7193a55178..53710fa3bd2 100644 --- a/src/Interpreters/InternalTextLogsQueue.h +++ b/src/Interpreters/InternalTextLogsQueue.h @@ -18,62 +18,6 @@ public: static Block getSampleBlock(); static MutableColumns getSampleColumns(); - template - bool push(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::push(std::forward(args)...); - } - - template - bool emplace(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::emplace(std::forward(args)...); - } - - template - bool pop(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::pop(std::forward(args)...); - } - - template - bool tryPush(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryPush(std::forward(args)...); - } - - template - bool tryEmplace(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryEmplace(std::forward(args)...); - } - - template - bool tryPop(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryPop(std::forward(args)...); - } - - template - void clear(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::clear(std::forward(args)...); - } - - template - void clearAndFinish(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::clearAndFinish(std::forward(args)...); - } - /// Is used to pass block from remote server to the client void pushBlock(Block && log_block); diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp new file mode 100644 index 00000000000..ef19eaa1c42 --- /dev/null +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -0,0 +1,75 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TABLE_IS_READ_ONLY; +} + + +BlockIO InterpreterCreateIndexQuery::execute() +{ + auto current_context = getContext(); + const auto & create_index = query_ptr->as(); + + AccessRightsElements required_access; + required_access.emplace_back(AccessType::ALTER_ADD_INDEX, create_index.getDatabase(), create_index.getTable()); + + if (!create_index.cluster.empty()) + { + DDLQueryOnClusterParams params; + params.access_to_check = std::move(required_access); + return executeDDLQueryOnCluster(query_ptr, current_context, params); + } + + current_context->checkAccess(required_access); + auto table_id = current_context->resolveStorageID(create_index, Context::ResolveOrdinary); + query_ptr->as().setDatabase(table_id.database_name); + + DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + if (typeid_cast(database.get()) + && !current_context->getClientInfo().is_replicated_database_internal) + { + auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); + guard->releaseTableLock(); + return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); + } + + StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); + if (table->isStaticStorage()) + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); + + /// Convert ASTCreateIndexQuery to AlterCommand. + AlterCommands alter_commands; + + AlterCommand command; + command.ast = create_index.convertToASTAlterCommand(); + command.index_decl = create_index.index_decl; + command.type = AlterCommand::ADD_INDEX; + command.index_name = create_index.index_name->as().name(); + command.if_not_exists = create_index.if_not_exists; + + alter_commands.emplace_back(std::move(command)); + + auto alter_lock = table->lockForAlter(current_context->getSettingsRef().lock_acquire_timeout); + StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); + alter_commands.validate(table, current_context); + alter_commands.prepare(metadata); + table->checkAlterIsPossible(alter_commands, current_context); + table->alter(alter_commands, current_context, alter_lock); + + return {}; +} + +} diff --git a/src/Interpreters/InterpreterCreateIndexQuery.h b/src/Interpreters/InterpreterCreateIndexQuery.h new file mode 100644 index 00000000000..63eaaf5f1c2 --- /dev/null +++ b/src/Interpreters/InterpreterCreateIndexQuery.h @@ -0,0 +1,24 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + +class InterpreterCreateIndexQuery : public IInterpreter, WithContext +{ +public: + InterpreterCreateIndexQuery(const ASTPtr & query_ptr_, ContextPtr context_) + : WithContext(context_) + , query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index b29f7372d38..7a00bbf524c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -150,10 +150,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// When attaching old-style database during server startup, we must always use Ordinary engine if (create.attach) throw Exception("Database engine must be specified for ATTACH DATABASE query", ErrorCodes::UNKNOWN_DATABASE_ENGINE); - bool old_style_database = getContext()->getSettingsRef().default_database_engine.value == DefaultDatabaseEngine::Ordinary; auto engine = std::make_shared(); auto storage = std::make_shared(); - engine->name = old_style_database ? "Ordinary" : "Atomic"; + engine->name = "Atomic"; engine->no_empty_args = true; storage->set(storage->engine, engine); create.set(create.storage, storage); @@ -196,8 +195,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (create_from_user) { - const auto & default_engine = getContext()->getSettingsRef().default_database_engine.value; - if (create.uuid == UUIDHelpers::Nil && default_engine == DefaultDatabaseEngine::Atomic) + if (create.uuid == UUIDHelpers::Nil) create.uuid = UUIDHelpers::generateV4(); /// Will enable Atomic engine for nested database } else if (attach_from_user && create.uuid == UUIDHelpers::Nil) @@ -249,13 +247,22 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } + bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); + bool need_lock_uuid = internal || need_write_metadata; + + /// Lock uuid, so we will known it's already in use. + /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); + TemporaryLockForUUIDDirectory uuid_lock; + if (need_lock_uuid) + uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; + else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); if (create.uuid != UUIDHelpers::Nil) create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER); - bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); - if (need_write_metadata) { create.attach = true; @@ -1031,6 +1038,10 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) create.attach = true; create.attach_short_syntax = true; create.if_not_exists = if_not_exists; + + /// Compatibility setting which should be enabled by default on attach + /// Otherwise server will be unable to start for some old-format of IPv6/IPv4 types + getContext()->setSetting("cast_ipv4_ipv6_default_on_conversion_error", 1); } /// TODO throw exception if !create.attach_short_syntax && !create.attach_from_path && !internal @@ -1163,70 +1174,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties) { - std::unique_ptr guard; - - String data_path; - DatabasePtr database; - - bool need_add_to_database = !create.temporary; - if (need_add_to_database) - { - /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). - * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. - */ - guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - - database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); - assertOrSetUUID(create, database); - - String storage_name = create.is_dictionary ? "Dictionary" : "Table"; - auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; - - /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. - if (database->isTableExist(create.getTable(), getContext())) - { - /// TODO Check structure of table - if (create.if_not_exists) - return false; - else if (create.replace_view) - { - /// when executing CREATE OR REPLACE VIEW, drop current existing view - auto drop_ast = std::make_shared(); - drop_ast->setDatabase(create.getDatabase()); - drop_ast->setTable(create.getTable()); - drop_ast->no_ddl_lock = true; - - auto drop_context = Context::createCopy(context); - InterpreterDropQuery interpreter(drop_ast, drop_context); - interpreter.execute(); - } - else - throw Exception(storage_already_exists_error_code, - "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); - } - else if (!create.attach) - { - /// Checking that table may exists in detached/detached permanently state - try - { - database->checkMetadataFilenameAvailability(create.getTable()); - } - catch (const Exception &) - { - if (create.if_not_exists) - return false; - throw; - } - } - - - data_path = database->getTableDataPath(create); - - if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) - throw Exception(storage_already_exists_error_code, - "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); - } - else + if (create.temporary) { if (create.if_not_exists && getContext()->tryResolveStorageID({"", create.getTable()}, Context::ResolveExternal)) return false; @@ -1237,6 +1185,65 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, return true; } + std::unique_ptr guard; + + String data_path; + DatabasePtr database; + + /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). + * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. + */ + guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); + + database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); + assertOrSetUUID(create, database); + + String storage_name = create.is_dictionary ? "Dictionary" : "Table"; + auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; + + /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. + if (database->isTableExist(create.getTable(), getContext())) + { + /// TODO Check structure of table + if (create.if_not_exists) + return false; + else if (create.replace_view) + { + /// when executing CREATE OR REPLACE VIEW, drop current existing view + auto drop_ast = std::make_shared(); + drop_ast->setDatabase(create.getDatabase()); + drop_ast->setTable(create.getTable()); + drop_ast->no_ddl_lock = true; + + auto drop_context = Context::createCopy(context); + InterpreterDropQuery interpreter(drop_ast, drop_context); + interpreter.execute(); + } + else + throw Exception(storage_already_exists_error_code, + "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); + } + else if (!create.attach) + { + /// Checking that table may exists in detached/detached permanently state + try + { + database->checkMetadataFilenameAvailability(create.getTable()); + } + catch (const Exception &) + { + if (create.if_not_exists) + return false; + throw; + } + } + + data_path = database->getTableDataPath(create); + + if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) + throw Exception(storage_already_exists_error_code, + "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); + bool from_path = create.attach_from_path.has_value(); String actual_data_path = data_path; if (from_path) @@ -1261,6 +1268,19 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, database->checkDetachedTableNotInUse(create.uuid); } + /// We should lock UUID on CREATE query (because for ATTACH it must be already locked previously). + /// But ATTACH without create.attach_short_syntax flag works like CREATE actually, that's why we check it. + bool need_lock_uuid = !create.attach_short_syntax; + TemporaryLockForUUIDDirectory uuid_lock; + if (need_lock_uuid) + uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; + else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) + { + /// FIXME MaterializedPostgreSQL works with UUIDs incorrectly and breaks invariants + if (database->getEngineName() != "MaterializedPostgreSQL") + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + } + StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) diff --git a/src/Interpreters/InterpreterDescribeCacheQuery.cpp b/src/Interpreters/InterpreterDescribeCacheQuery.cpp new file mode 100644 index 00000000000..dd6df26c6af --- /dev/null +++ b/src/Interpreters/InterpreterDescribeCacheQuery.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +static Block getSampleBlock() +{ + ColumnsWithTypeAndName columns{ + ColumnWithTypeAndName{std::make_shared(), "max_size"}, + ColumnWithTypeAndName{std::make_shared(), "max_elements"}, + ColumnWithTypeAndName{std::make_shared(), "max_file_segment_size"}, + ColumnWithTypeAndName{std::make_shared>(), "cache_on_write_operations"}, + ColumnWithTypeAndName{std::make_shared>(), "enable_cache_hits_threshold"}, + ColumnWithTypeAndName{std::make_shared(), "current_size"}, + ColumnWithTypeAndName{std::make_shared(), "current_elements"}, + ColumnWithTypeAndName{std::make_shared(), "path"} + }; + return Block(columns); +} + +BlockIO InterpreterDescribeCacheQuery::execute() +{ + getContext()->checkAccess(AccessType::SHOW_CACHES); + + const auto & ast = query_ptr->as(); + Block sample_block = getSampleBlock(); + MutableColumns res_columns = sample_block.cloneEmptyColumns(); + + auto cache_data = FileCacheFactory::instance().getByName(ast.cache_name); + const auto & settings = cache_data.settings; + const auto & cache = cache_data.cache; + + res_columns[0]->insert(settings.max_size); + res_columns[1]->insert(settings.max_elements); + res_columns[2]->insert(settings.max_file_segment_size); + res_columns[3]->insert(settings.cache_on_write_operations); + res_columns[4]->insert(settings.enable_cache_hits_threshold); + res_columns[5]->insert(cache->getUsedCacheSize()); + res_columns[6]->insert(cache->getFileSegmentsNum()); + res_columns[7]->insert(cache->getBasePath()); + + BlockIO res; + size_t num_rows = res_columns[0]->size(); + auto source = std::make_shared(sample_block, Chunk(std::move(res_columns), num_rows)); + res.pipeline = QueryPipeline(std::move(source)); + + return res; +} + +} diff --git a/src/Interpreters/InterpreterDescribeCacheQuery.h b/src/Interpreters/InterpreterDescribeCacheQuery.h new file mode 100644 index 00000000000..31b77f7d064 --- /dev/null +++ b/src/Interpreters/InterpreterDescribeCacheQuery.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class InterpreterDescribeCacheQuery : public IInterpreter, WithContext +{ +public: + InterpreterDescribeCacheQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_), query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + + +} diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp new file mode 100644 index 00000000000..2339e0dc68e --- /dev/null +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TABLE_IS_READ_ONLY; +} + + +BlockIO InterpreterDropIndexQuery::execute() +{ + auto current_context = getContext(); + const auto & drop_index = query_ptr->as(); + + AccessRightsElements required_access; + required_access.emplace_back(AccessType::ALTER_DROP_INDEX, drop_index.getDatabase(), drop_index.getTable()); + + if (!drop_index.cluster.empty()) + { + DDLQueryOnClusterParams params; + params.access_to_check = std::move(required_access); + return executeDDLQueryOnCluster(query_ptr, current_context, params); + } + + current_context->checkAccess(required_access); + auto table_id = current_context->resolveStorageID(drop_index, Context::ResolveOrdinary); + query_ptr->as().setDatabase(table_id.database_name); + + DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + if (typeid_cast(database.get()) + && !current_context->getClientInfo().is_replicated_database_internal) + { + auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); + guard->releaseTableLock(); + return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); + } + + StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); + if (table->isStaticStorage()) + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); + + /// Convert ASTDropIndexQuery to AlterCommand. + AlterCommands alter_commands; + + AlterCommand command; + command.ast = drop_index.convertToASTAlterCommand(); + command.type = AlterCommand::DROP_INDEX; + command.index_name = drop_index.index_name->as().name(); + command.if_exists = drop_index.if_exists; + + alter_commands.emplace_back(std::move(command)); + + auto alter_lock = table->lockForAlter(current_context->getSettingsRef().lock_acquire_timeout); + StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); + alter_commands.validate(table, current_context); + alter_commands.prepare(metadata); + table->checkAlterIsPossible(alter_commands, current_context); + table->alter(alter_commands, current_context, alter_lock); + + return {}; +} + +} diff --git a/src/Interpreters/InterpreterDropIndexQuery.h b/src/Interpreters/InterpreterDropIndexQuery.h new file mode 100644 index 00000000000..c6fb3add72f --- /dev/null +++ b/src/Interpreters/InterpreterDropIndexQuery.h @@ -0,0 +1,24 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + +class InterpreterDropIndexQuery : public IInterpreter, WithContext +{ +public: + InterpreterDropIndexQuery(const ASTPtr & query_ptr_, ContextPtr context_) + : WithContext(context_) + , query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 41b65e73efa..ac731ec6f4b 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -62,7 +62,7 @@ BlockIO InterpreterDropQuery::execute() } if (getContext()->getSettingsRef().database_atomic_wait_for_drop_and_detach_synchronously) - drop.no_delay = true; + drop.sync = true; if (drop.table) return executeToTable(drop); @@ -89,7 +89,7 @@ BlockIO InterpreterDropQuery::executeToTable(ASTDropQuery & query) DatabasePtr database; UUID table_to_wait_on = UUIDHelpers::Nil; auto res = executeToTableImpl(getContext(), query, database, table_to_wait_on); - if (query.no_delay) + if (query.sync) waitForTableToBeActuallyDroppedOrDetached(query, database, table_to_wait_on); return res; } @@ -244,7 +244,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue DatabaseCatalog::instance().tryRemoveLoadingDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, is_drop_or_detach_database); - database->dropTable(context_, table_id.table_name, query.no_delay); + database->dropTable(context_, table_id.table_name, query.sync); } db = database; @@ -300,7 +300,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) } catch (...) { - if (query.no_delay) + if (query.sync) { for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); @@ -308,7 +308,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) throw; } - if (query.no_delay) + if (query.sync) { for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); @@ -345,7 +345,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, query_for_table.kind = query.kind; query_for_table.if_exists = true; query_for_table.setDatabase(database_name); - query_for_table.no_delay = query.no_delay; + query_for_table.sync = query.sync; /// Flush should not be done if shouldBeEmptyOnDetach() == false, /// since in this case getTablesIterator() may do some additional work, @@ -368,7 +368,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, } } - if (!drop && query.no_delay) + if (!drop && query.sync) { /// Avoid "some tables are still in use" when sync mode is enabled for (const auto & table_uuid : uuids_to_wait) @@ -428,7 +428,7 @@ void InterpreterDropQuery::extendQueryLogElemImpl(QueryLogElement & elem, const elem.query_kind = "Drop"; } -void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool no_delay) +void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync) { if (DatabaseCatalog::instance().tryGetTable(target_table_id, current_context)) { @@ -437,7 +437,7 @@ void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr drop_query->setDatabase(target_table_id.database_name); drop_query->setTable(target_table_id.table_name); drop_query->kind = kind; - drop_query->no_delay = no_delay; + drop_query->sync = sync; drop_query->if_exists = true; ASTPtr ast_drop_query = drop_query; /// FIXME We have to use global context to execute DROP query for inner table diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 1a38abcdff9..2b65039954b 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -26,7 +26,7 @@ public: void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr) const override; - static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool no_delay); + static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync); private: AccessRightsElements getRequiredAccessForDDLOnCluster() const; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 5dcee1eae05..6b081467ae7 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -35,15 +37,19 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include +#include #include +#include #include #include #include @@ -200,6 +206,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); @@ -292,6 +302,14 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 1bf188596d7..cf229442442 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1176,7 +1176,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

row_level_filter, expressions.prewhere_info->row_level_column_name, - false); + true); row_level_filter_step->setStepDescription("Row-level security filter (PREWHERE)"); query_plan.addStep(std::move(row_level_filter_step)); @@ -1242,8 +1242,12 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( - params, - final, - /* only_merge_= */ false); + Aggregator::Params params(keys, aggregates, overflow_row, settings.max_threads); auto merging_aggregated = std::make_unique( query_plan.getCurrentDataStream(), - std::move(transform_params), + params, + final, settings.distributed_aggregation_memory_efficient && is_remote_storage, settings.max_threads, - settings.aggregation_memory_efficient_merge_threads); + settings.aggregation_memory_efficient_merge_threads, + should_produce_results_in_order_of_bucket_number); query_plan.addStep(std::move(merging_aggregated)); } @@ -1688,6 +1686,9 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( query_plan.addStep(std::move(expression_before_aggregation)); } + // Let's just choose the safe option since we don't know the value of `to_stage` here. + const bool should_produce_results_in_order_of_bucket_number = true; + executeMergeAggregatedImpl( query_plan, query_info.projection->aggregate_overflow_row, @@ -1696,7 +1697,8 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( false, context_->getSettingsRef(), query_info.projection->aggregation_keys, - query_info.projection->aggregate_descriptions); + query_info.projection->aggregate_descriptions, + should_produce_results_in_order_of_bucket_number); } } } @@ -2174,11 +2176,12 @@ static Aggregator::Params getAggregatorParams( const ASTPtr & query_ptr, const SelectQueryExpressionAnalyzer & query_analyzer, const Context & context, - const Block & current_data_stream_header, - const ColumnNumbers & keys, + const Names & keys, const AggregateDescriptions & aggregates, - bool overflow_row, const Settings & settings, - size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes) + bool overflow_row, + const Settings & settings, + size_t group_by_two_level_threshold, + size_t group_by_two_level_threshold_bytes) { const auto stats_collecting_params = Aggregator::Params::StatsCollectingParams( query_ptr, @@ -2186,8 +2189,8 @@ static Aggregator::Params getAggregatorParams( settings.max_entries_for_hash_table_stats, settings.max_size_to_preallocate_for_aggregation); - return Aggregator::Params{ - current_data_stream_header, + return Aggregator::Params + { keys, aggregates, overflow_row, @@ -2204,42 +2207,30 @@ static Aggregator::Params getAggregatorParams( settings.min_free_disk_space_for_temporary_data, settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression, - Block{}, + /* only_merge */ false, stats_collecting_params }; } -static GroupingSetsParamsList getAggregatorGroupingSetsParams( - const SelectQueryExpressionAnalyzer & query_analyzer, - const Block & header_before_aggregation, - const ColumnNumbers & all_keys -) +static GroupingSetsParamsList getAggregatorGroupingSetsParams(const SelectQueryExpressionAnalyzer & query_analyzer, const Names & all_keys) { GroupingSetsParamsList result; if (query_analyzer.useGroupingSetKey()) { auto const & aggregation_keys_list = query_analyzer.aggregationKeysList(); - ColumnNumbersList grouping_sets_with_keys; - ColumnNumbersList missing_columns_per_set; - for (const auto & aggregation_keys : aggregation_keys_list) { - ColumnNumbers keys; - std::unordered_set keys_set; + NameSet keys; for (const auto & key : aggregation_keys) - { - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - keys_set.insert(keys.back()); - } + keys.insert(key.name); - ColumnNumbers missing_indexes; - for (size_t i = 0; i < all_keys.size(); ++i) - { - if (!keys_set.contains(all_keys[i])) - missing_indexes.push_back(i); - } - result.emplace_back(std::move(keys), std::move(missing_indexes)); + Names missing_keys; + for (const auto & key : all_keys) + if (!keys.contains(key)) + missing_keys.push_back(key); + + result.emplace_back(aggregation_keys.getNames(), std::move(missing_keys)); } } return result; @@ -2254,24 +2245,24 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac if (options.is_projection_query) return; - const auto & header_before_aggregation = query_plan.getCurrentDataStream().header; - AggregateDescriptions aggregates = query_analyzer->aggregates(); - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); const Settings & settings = context->getSettingsRef(); - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(header_before_aggregation.getPositionByName(key.name)); + const auto & keys = query_analyzer->aggregationKeys().getNames(); - auto aggregator_params = getAggregatorParams(query_ptr, *query_analyzer, *context, header_before_aggregation, keys, aggregates, overflow_row, settings, - settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); + auto aggregator_params = getAggregatorParams( + query_ptr, + *query_analyzer, + *context, + keys, + aggregates, + overflow_row, + settings, + settings.group_by_two_level_threshold, + settings.group_by_two_level_threshold_bytes); - auto grouping_sets_params = getAggregatorGroupingSetsParams(*query_analyzer, header_before_aggregation, keys); + auto grouping_sets_params = getAggregatorGroupingSetsParams(*query_analyzer, keys); SortDescription group_by_sort_description; @@ -2287,19 +2278,22 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); + const bool should_produce_results_in_order_of_bucket_number + = options.to_stage == QueryProcessingStage::WithMergeableState && settings.distributed_aggregation_memory_efficient; + auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), std::move(aggregator_params), std::move(grouping_sets_params), final, - /* only_merge_= */ false, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, merge_threads, temporary_data_merge_threads, storage_has_evenly_distributed_read, std::move(group_by_info), - std::move(group_by_sort_description)); + std::move(group_by_sort_description), + should_produce_results_in_order_of_bucket_number); query_plan.addStep(std::move(aggregating_step)); } @@ -2312,6 +2306,9 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) return; + const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState + && context->getSettingsRef().distributed_aggregation_memory_efficient; + executeMergeAggregatedImpl( query_plan, overflow_row, @@ -2320,7 +2317,8 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool has_grouping_sets, context->getSettingsRef(), query_analyzer->aggregationKeys(), - query_analyzer->aggregates()); + query_analyzer->aggregates(), + should_produce_results_in_order_of_bucket_number); } @@ -2339,11 +2337,9 @@ void InterpreterSelectQuery::executeTotalsAndHaving( { const Settings & settings = context->getSettingsRef(); - const auto & header_before = query_plan.getCurrentDataStream().header; - auto totals_having_step = std::make_unique( query_plan.getCurrentDataStream(), - getAggregatesMask(header_before, query_analyzer->aggregates()), + query_analyzer->aggregates(), overflow_row, expression, has_having ? getSelectQuery().having()->getColumnName() : "", @@ -2357,25 +2353,23 @@ void InterpreterSelectQuery::executeTotalsAndHaving( void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modificator modificator) { - const auto & header_before_transform = query_plan.getCurrentDataStream().header; - const Settings & settings = context->getSettingsRef(); - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(header_before_transform.getPositionByName(key.name)); + const auto & keys = query_analyzer->aggregationKeys().getNames(); - auto params = getAggregatorParams(query_ptr, *query_analyzer, *context, header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0); - auto transform_params = std::make_shared( - std::move(params), - /* final_= */ true, - /* only_merge_= */ false); + // Arguments will not be present in Rollup / Cube input header and they don't actually needed 'cause these steps will work with AggregateFunctionState-s anyway. + auto aggregates = query_analyzer->aggregates(); + for (auto & aggregate : aggregates) + aggregate.argument_names.clear(); + + auto params = getAggregatorParams(query_ptr, *query_analyzer, *context, keys, aggregates, false, settings, 0, 0); + const bool final = true; QueryPlanStepPtr step; if (modificator == Modificator::ROLLUP) - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final); else if (modificator == Modificator::CUBE) - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final); query_plan.addStep(std::move(step)); } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index f0150fe663f..e886027683f 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -52,10 +52,6 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_children) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - /// This is required for UNION to match headers correctly. - if (num_children > 1) - options.reorderColumns(); - /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, /// because names could be different. @@ -163,7 +159,22 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( { Blocks headers(num_children); for (size_t query_num = 0; query_num < num_children; ++query_num) + { headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); + const auto & current_required_result_column_names = required_result_column_names_for_other_selects[query_num]; + if (!current_required_result_column_names.empty()) + { + const auto & header_columns = headers[query_num].getNames(); + if (current_required_result_column_names != header_columns) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Different order of columns in UNION subquery: {} and {}", + fmt::join(current_required_result_column_names, ", "), + fmt::join(header_columns, ", ")); + } + + } + } result_header = getCommonHeaderForUnion(headers); } diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 609df1404ca..d623b6c71d6 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -5,6 +5,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include @@ -142,6 +147,24 @@ String InterpreterShowTablesQuery::getRewrittenQuery() BlockIO InterpreterShowTablesQuery::execute() { + const auto & query = query_ptr->as(); + if (query.caches) + { + getContext()->checkAccess(AccessType::SHOW_CACHES); + + Block sample_block{ColumnWithTypeAndName(std::make_shared(), "Caches")}; + MutableColumns res_columns = sample_block.cloneEmptyColumns(); + auto caches = FileCacheFactory::instance().getAllByName(); + for (const auto & [name, _] : caches) + res_columns[0]->insert(name); + BlockIO res; + size_t num_rows = res_columns[0]->size(); + auto source = std::make_shared(sample_block, Chunk(std::move(res_columns), num_rows)); + res.pipeline = QueryPipeline(std::move(source)); + + return res; + } + return executeQuery(getRewrittenQuery(), getContext(), true); } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9196a5222a2..695ea53e65e 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -312,11 +312,12 @@ BlockIO InterpreterSystemQuery::execute() #endif case Type::DROP_FILESYSTEM_CACHE: { + getContext()->checkAccess(AccessType::SYSTEM_DROP_FILESYSTEM_CACHE); if (query.filesystem_cache_path.empty()) { auto caches = FileCacheFactory::instance().getAll(); for (const auto & [_, cache_data] : caches) - cache_data.cache->removeIfReleasable(/* remove_persistent_files */false); + cache_data->cache->removeIfReleasable(/* remove_persistent_files */false); } else { diff --git a/src/Interpreters/InterpreterTransactionControlQuery.h b/src/Interpreters/InterpreterTransactionControlQuery.h index bf2dc7891a7..a66a740ce0c 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.h +++ b/src/Interpreters/InterpreterTransactionControlQuery.h @@ -20,7 +20,6 @@ public: bool ignoreLimits() const override { return true; } bool supportsTransactions() const override { return true; } -private: BlockIO executeBegin(ContextMutablePtr session_context); BlockIO executeCommit(ContextMutablePtr session_context); static BlockIO executeRollback(ContextMutablePtr session_context); diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 3dd6b7de574..9ddd4ac0be0 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -580,7 +580,7 @@ void MergeJoin::mergeRightBlocks() void MergeJoin::mergeInMemoryRightBlocks() { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (right_blocks.empty()) return; @@ -613,7 +613,7 @@ void MergeJoin::mergeInMemoryRightBlocks() void MergeJoin::mergeFlushedRightBlocks() { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); auto callback = [&](const Block & block) { @@ -638,7 +638,7 @@ bool MergeJoin::saveRightBlock(Block && block) { if (is_in_memory) { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (!is_in_memory) { diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp index e6b4818b4d7..432116feaf5 100644 --- a/src/Interpreters/MergeTreeTransaction.cpp +++ b/src/Interpreters/MergeTreeTransaction.cpp @@ -24,16 +24,17 @@ static TableLockHolder getLockForOrdinary(const StoragePtr & storage) return storage->lockForShare(RWLockImpl::NO_QUERY, default_timeout); } -MergeTreeTransaction::MergeTreeTransaction(CSN snapshot_, LocalTID local_tid_, UUID host_id) +MergeTreeTransaction::MergeTreeTransaction(CSN snapshot_, LocalTID local_tid_, UUID host_id, std::list::iterator snapshot_it_) : tid({snapshot_, local_tid_, host_id}) , snapshot(snapshot_) + , snapshot_in_use_it(snapshot_it_) , csn(Tx::UnknownCSN) { } void MergeTreeTransaction::setSnapshot(CSN new_snapshot) { - snapshot = new_snapshot; + snapshot.store(new_snapshot, std::memory_order_relaxed); } MergeTreeTransaction::State MergeTreeTransaction::getState() const @@ -219,19 +220,31 @@ void MergeTreeTransaction::afterCommit(CSN assigned_csn) noexcept /// It's not a problem if server crash before CSN is written, because we already have TID in data part and entry in the log. [[maybe_unused]] CSN prev_value = csn.exchange(assigned_csn); chassert(prev_value == Tx::CommittingCSN); - for (const auto & part : creating_parts) + + DataPartsVector created_parts; + DataPartsVector removed_parts; + RunningMutationsList committed_mutations; + { + /// We don't really need mutex here, because no concurrent modifications of transaction object may happen after commit. + std::lock_guard lock{mutex}; + created_parts = creating_parts; + removed_parts = removing_parts; + committed_mutations = mutations; + } + + for (const auto & part : created_parts) { part->version.creation_csn.store(csn); part->appendCSNToVersionMetadata(VersionMetadata::WhichCSN::CREATION); } - for (const auto & part : removing_parts) + for (const auto & part : removed_parts) { part->version.removal_csn.store(csn); part->appendCSNToVersionMetadata(VersionMetadata::WhichCSN::REMOVAL); } - for (const auto & storage_and_mutation : mutations) + for (const auto & storage_and_mutation : committed_mutations) storage_and_mutation.first->setMutationCSN(storage_and_mutation.second, csn); } @@ -313,7 +326,7 @@ void MergeTreeTransaction::onException() String MergeTreeTransaction::dumpDescription() const { - String res = fmt::format("{} state: {}, snapshot: {}", tid, getState(), snapshot); + String res = fmt::format("{} state: {}, snapshot: {}", tid, getState(), getSnapshot()); if (isReadOnly()) { @@ -335,7 +348,7 @@ String MergeTreeTransaction::dumpDescription() const { String info = fmt::format("{} (created by {}, {})", part->name, part->version.getCreationTID(), part->version.creation_csn); std::get<1>(storage_to_changes[&(part->storage)]).push_back(std::move(info)); - chassert(!part->version.creation_csn || part->version.creation_csn <= snapshot); + chassert(!part->version.creation_csn || part->version.creation_csn <= getSnapshot()); } for (const auto & mutation : mutations) diff --git a/src/Interpreters/MergeTreeTransaction.h b/src/Interpreters/MergeTreeTransaction.h index 8b537a7bcd8..7397ea12c12 100644 --- a/src/Interpreters/MergeTreeTransaction.h +++ b/src/Interpreters/MergeTreeTransaction.h @@ -31,13 +31,13 @@ public: ROLLED_BACK, }; - CSN getSnapshot() const { return snapshot; } + CSN getSnapshot() const { return snapshot.load(std::memory_order_relaxed); } void setSnapshot(CSN new_snapshot); State getState() const; const TransactionID tid; - MergeTreeTransaction(CSN snapshot_, LocalTID local_tid_, UUID host_id); + MergeTreeTransaction(CSN snapshot_, LocalTID local_tid_, UUID host_id, std::list::iterator snapshot_it_); void addNewPart(const StoragePtr & storage, const DataPartPtr & new_part); void removeOldPart(const StoragePtr & storage, const DataPartPtr & part_to_remove, const TransactionInfoContext & context); @@ -71,16 +71,16 @@ private: Stopwatch elapsed; /// Usually it's equal to tid.start_csn, but can be changed by SET SNAPSHOT query (for introspection purposes and time-traveling) - CSN snapshot; - std::list::iterator snapshot_in_use_it; + std::atomic snapshot; + const std::list::iterator snapshot_in_use_it; /// Lists of changes made by transaction - std::unordered_set storages; - std::vector table_read_locks_for_ordinary_db; - DataPartsVector creating_parts; - DataPartsVector removing_parts; + std::unordered_set storages TSA_GUARDED_BY(mutex); + std::vector table_read_locks_for_ordinary_db TSA_GUARDED_BY(mutex); + DataPartsVector creating_parts TSA_GUARDED_BY(mutex); + DataPartsVector removing_parts TSA_GUARDED_BY(mutex); using RunningMutationsList = std::vector>; - RunningMutationsList mutations; + RunningMutationsList mutations TSA_GUARDED_BY(mutex); std::atomic csn; }; diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index 97feed54c08..2a18c2c700a 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include #include diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index 032befe1993..6b5a6a7f8eb 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -31,8 +31,6 @@ struct SelectQueryOptions bool only_analyze = false; bool modify_inplace = false; bool remove_duplicates = false; - /// This is required for UNION to match headers correctly. - bool reorder_columns_as_required_header = false; bool ignore_quota = false; bool ignore_limits = false; /// This flag is needed to analyze query ignoring table projections. @@ -99,12 +97,6 @@ struct SelectQueryOptions return *this; } - SelectQueryOptions & reorderColumns(bool value = true) - { - reorder_columns_as_required_header = value; - return *this; - } - SelectQueryOptions & noSubquery() { subquery_depth = 0; diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 28bbea54110..ab443f58cf2 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -103,7 +103,7 @@ void NO_INLINE Set::insertFromBlockImplCase( void Set::setHeader(const ColumnsWithTypeAndName & header) { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (!data.empty()) return; diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index 29ba24c2e4c..c60c3aec9c6 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -10,7 +10,7 @@ namespace Poco { namespace Util { -class AbstractConfiguration; +class AbstractConfiguration; // NOLINT(cppcoreguidelines-virtual-class-destructor) } } diff --git a/src/Interpreters/TransactionLog.cpp b/src/Interpreters/TransactionLog.cpp index 4f79aaff2c7..a08f940a748 100644 --- a/src/Interpreters/TransactionLog.cpp +++ b/src/Interpreters/TransactionLog.cpp @@ -43,16 +43,13 @@ catch (...) TransactionLog::TransactionLog() - : log(&Poco::Logger::get("TransactionLog")) + : global_context(Context::getGlobalContextInstance()) + , log(&Poco::Logger::get("TransactionLog")) + , zookeeper_path(global_context->getConfigRef().getString("transaction_log.zookeeper_path", "/clickhouse/txn")) + , zookeeper_path_log(zookeeper_path + "/log") + , fault_probability_before_commit(global_context->getConfigRef().getDouble("transaction_log.fault_probability_before_commit", 0)) + , fault_probability_after_commit(global_context->getConfigRef().getDouble("transaction_log.fault_probability_after_commit", 0)) { - global_context = Context::getGlobalContextInstance(); - global_context->checkTransactionsAreAllowed(); - - zookeeper_path = global_context->getConfigRef().getString("transaction_log.zookeeper_path", "/clickhouse/txn"); - zookeeper_path_log = zookeeper_path + "/log"; - fault_probability_before_commit = global_context->getConfigRef().getDouble("transaction_log.fault_probability_before_commit", 0); - fault_probability_after_commit = global_context->getConfigRef().getDouble("transaction_log.fault_probability_after_commit", 0); - loadLogFromZooKeeper(); updating_thread = ThreadFromGlobalPool(&TransactionLog::runUpdatingThread, this); @@ -128,7 +125,7 @@ void TransactionLog::loadEntries(Strings::const_iterator beg, Strings::const_ite LOG_TRACE(log, "Loading {} entries from {}: {}..{}", entries_count, zookeeper_path_log, *beg, last_entry); futures.reserve(entries_count); for (auto it = beg; it != end; ++it) - futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path_log) / *it)); + futures.emplace_back(TSA_READ_ONE_THREAD(zookeeper)->asyncGet(fs::path(zookeeper_path_log) / *it)); std::vector> loaded; loaded.reserve(entries_count); @@ -213,7 +210,7 @@ void TransactionLog::runUpdatingThread() try { /// Do not wait if we have some transactions to finalize - if (unknown_state_list_loaded.empty()) + if (TSA_READ_ONE_THREAD(unknown_state_list_loaded).empty()) log_updated_event->wait(); if (stop_flag.load()) @@ -230,7 +227,7 @@ void TransactionLog::runUpdatingThread() /// It's possible that we connected to different [Zoo]Keeper instance /// so we may read a bit stale state. - zookeeper->sync(zookeeper_path_log); + TSA_READ_ONE_THREAD(zookeeper)->sync(zookeeper_path_log); } loadNewEntries(); @@ -255,13 +252,13 @@ void TransactionLog::runUpdatingThread() void TransactionLog::loadNewEntries() { - Strings entries_list = zookeeper->getChildren(zookeeper_path_log, nullptr, log_updated_event); + Strings entries_list = TSA_READ_ONE_THREAD(zookeeper)->getChildren(zookeeper_path_log, nullptr, log_updated_event); chassert(!entries_list.empty()); ::sort(entries_list.begin(), entries_list.end()); - auto it = std::upper_bound(entries_list.begin(), entries_list.end(), last_loaded_entry); + auto it = std::upper_bound(entries_list.begin(), entries_list.end(), TSA_READ_ONE_THREAD(last_loaded_entry)); loadEntries(it, entries_list.end()); - chassert(last_loaded_entry == entries_list.back()); - chassert(latest_snapshot == deserializeCSN(last_loaded_entry)); + chassert(TSA_READ_ONE_THREAD(last_loaded_entry) == entries_list.back()); + chassert(latest_snapshot == deserializeCSN(TSA_READ_ONE_THREAD(last_loaded_entry))); latest_snapshot.notify_all(); } @@ -281,7 +278,7 @@ void TransactionLog::removeOldEntries() /// TODO we will need a bit more complex logic for multiple hosts Coordination::Stat stat; - CSN old_tail_ptr = deserializeCSN(zookeeper->get(zookeeper_path + "/tail_ptr", &stat)); + CSN old_tail_ptr = deserializeCSN(TSA_READ_ONE_THREAD(zookeeper)->get(zookeeper_path + "/tail_ptr", &stat)); CSN new_tail_ptr = getOldestSnapshot(); if (new_tail_ptr < old_tail_ptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected tail_ptr {}, oldest snapshot is {}, it's a bug", old_tail_ptr, new_tail_ptr); @@ -290,7 +287,7 @@ void TransactionLog::removeOldEntries() /// (it's not supposed to fail with ZBADVERSION while there is only one host) LOG_TRACE(log, "Updating tail_ptr from {} to {}", old_tail_ptr, new_tail_ptr); - zookeeper->set(zookeeper_path + "/tail_ptr", serializeCSN(new_tail_ptr), stat.version); + TSA_READ_ONE_THREAD(zookeeper)->set(zookeeper_path + "/tail_ptr", serializeCSN(new_tail_ptr), stat.version); tail_ptr.store(new_tail_ptr); /// Now we can find and remove old entries @@ -314,7 +311,7 @@ void TransactionLog::removeOldEntries() continue; LOG_TEST(log, "Removing entry {} -> {}", elem.second.tid, elem.second.csn); - auto code = zookeeper->tryRemove(zookeeper_path_log + "/" + serializeCSN(elem.second.csn)); + auto code = TSA_READ_ONE_THREAD(zookeeper)->tryRemove(zookeeper_path_log + "/" + serializeCSN(elem.second.csn)); if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE) removed_entries.push_back(elem.first); } @@ -376,11 +373,11 @@ MergeTreeTransactionPtr TransactionLog::beginTransaction() std::lock_guard lock{running_list_mutex}; CSN snapshot = latest_snapshot.load(); LocalTID ltid = 1 + local_tid_counter.fetch_add(1); - txn = std::make_shared(snapshot, ltid, ServerUUID::get()); + auto snapshot_lock = snapshots_in_use.insert(snapshots_in_use.end(), snapshot); + txn = std::make_shared(snapshot, ltid, ServerUUID::get(), snapshot_lock); bool inserted = running_list.try_emplace(txn->tid.getHash(), txn).second; if (!inserted) throw Exception(ErrorCodes::LOGICAL_ERROR, "I's a bug: TID {} {} exists", txn->tid.getHash(), txn->tid); - txn->snapshot_in_use_it = snapshots_in_use.insert(snapshots_in_use.end(), snapshot); } LOG_TEST(log, "Beginning transaction {} ({})", txn->tid, txn->tid.getHash()); @@ -595,7 +592,7 @@ TransactionLog::TransactionsList TransactionLog::getTransactionsList() const void TransactionLog::sync() const { - Strings entries_list = zookeeper->getChildren(zookeeper_path_log); + Strings entries_list = getZooKeeper()->getChildren(zookeeper_path_log); chassert(!entries_list.empty()); ::sort(entries_list.begin(), entries_list.end()); CSN newest_csn = deserializeCSN(entries_list.back()); diff --git a/src/Interpreters/TransactionLog.h b/src/Interpreters/TransactionLog.h index 49aa77b9868..71a5d3a6fc6 100644 --- a/src/Interpreters/TransactionLog.h +++ b/src/Interpreters/TransactionLog.h @@ -129,7 +129,7 @@ public: void sync() const; private: - void loadLogFromZooKeeper(); + void loadLogFromZooKeeper() TSA_REQUIRES(mutex); void runUpdatingThread(); void loadEntries(Strings::const_iterator beg, Strings::const_iterator end); @@ -149,8 +149,8 @@ private: CSN getCSNImpl(const TIDHash & tid_hash) const; - ContextPtr global_context; - Poco::Logger * log; + const ContextPtr global_context; + Poco::Logger * const log; /// The newest snapshot available for reading std::atomic latest_snapshot; @@ -167,24 +167,24 @@ private: TransactionID tid; }; using TIDMap = std::unordered_map; - TIDMap tid_to_csn; + TIDMap tid_to_csn TSA_GUARDED_BY(mutex); mutable std::mutex running_list_mutex; /// Transactions that are currently processed - TransactionsList running_list; + TransactionsList running_list TSA_GUARDED_BY(running_list_mutex); /// If we lost connection on attempt to create csn- node then we don't know transaction's state. using UnknownStateList = std::vector>; - UnknownStateList unknown_state_list; - UnknownStateList unknown_state_list_loaded; + UnknownStateList unknown_state_list TSA_GUARDED_BY(running_list_mutex); + UnknownStateList unknown_state_list_loaded TSA_GUARDED_BY(running_list_mutex); /// Ordered list of snapshots that are currently used by some transactions. Needed for background cleanup. - std::list snapshots_in_use; + std::list snapshots_in_use TSA_GUARDED_BY(running_list_mutex); - ZooKeeperPtr zookeeper; - String zookeeper_path; + ZooKeeperPtr zookeeper TSA_GUARDED_BY(mutex); + const String zookeeper_path; - String zookeeper_path_log; + const String zookeeper_path_log; /// Name of the newest entry that was loaded from log in ZK - String last_loaded_entry; + String last_loaded_entry TSA_GUARDED_BY(mutex); /// The oldest CSN such that we store in log entries with TransactionIDs containing this CSN. std::atomic tail_ptr = Tx::UnknownCSN; @@ -193,8 +193,8 @@ private: std::atomic_bool stop_flag = false; ThreadFromGlobalPool updating_thread; - Float64 fault_probability_before_commit = 0; - Float64 fault_probability_after_commit = 0; + const Float64 fault_probability_before_commit = 0; + const Float64 fault_probability_after_commit = 0; }; template diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 3d14955c16a..9a4a956f3dc 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -429,21 +429,28 @@ void renameDuplicatedColumns(const ASTSelectQuery * select_query) /// This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result. /// Also we have to remove duplicates in case of GLOBAL subqueries. Their results are placed into tables so duplicates are impossible. /// Also remove all INTERPOLATE columns which are not in SELECT anymore. -void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups, bool reorder_columns_as_required_header) +void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups) { ASTs & elements = select_query->select()->children; - std::map required_columns_with_duplicate_count; + std::unordered_map required_columns_with_duplicate_count; + /// Order of output columns should match order in required_result_columns, + /// otherwise UNION queries may have incorrect header when subselect has duplicated columns. + /// + /// NOTE: multimap is required since there can be duplicated column names. + std::unordered_multimap output_columns_positions; if (!required_result_columns.empty()) { /// Some columns may be queried multiple times, like SELECT x, y, y FROM table. - for (const auto & name : required_result_columns) + for (size_t i = 0; i < required_result_columns.size(); ++i) { + const auto & name = required_result_columns[i]; if (remove_dups) required_columns_with_duplicate_count[name] = 1; else ++required_columns_with_duplicate_count[name]; + output_columns_positions.emplace(name, i); } } else if (remove_dups) @@ -455,49 +462,44 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const else return; - ASTs new_elements; - new_elements.reserve(elements.size()); + ASTs new_elements(elements.size() + output_columns_positions.size()); + size_t new_elements_size = 0; NameSet remove_columns; - /// Resort columns according to required_result_columns. - if (reorder_columns_as_required_header && !required_result_columns.empty()) - { - std::unordered_map name_pos; - { - size_t pos = 0; - for (const auto & name : required_result_columns) - name_pos[name] = pos++; - } - ::sort(elements.begin(), elements.end(), [&](const auto & lhs, const auto & rhs) - { - String lhs_name = lhs->getAliasOrColumnName(); - String rhs_name = rhs->getAliasOrColumnName(); - size_t lhs_pos = name_pos.size(); - size_t rhs_pos = name_pos.size(); - if (auto it = name_pos.find(lhs_name); it != name_pos.end()) - lhs_pos = it->second; - if (auto it = name_pos.find(rhs_name); it != name_pos.end()) - rhs_pos = it->second; - return lhs_pos < rhs_pos; - }); - } - for (const auto & elem : elements) { String name = elem->getAliasOrColumnName(); + /// Columns that are presented in output_columns_positions should + /// appears in the same order in the new_elements, hence default + /// result_index goes after all elements of output_columns_positions + /// (it is for columns that are not located in + /// output_columns_positions, i.e. untuple()) + size_t result_index = output_columns_positions.size() + new_elements_size; + + /// Note, order of duplicated columns is not important here (since they + /// are the same), only order for unique columns is important, so it is + /// fine to use multimap here. + if (auto it = output_columns_positions.find(name); it != output_columns_positions.end()) + { + result_index = it->second; + output_columns_positions.erase(it); + } + auto it = required_columns_with_duplicate_count.find(name); if (required_columns_with_duplicate_count.end() != it && it->second) { - new_elements.push_back(elem); + new_elements[result_index] = elem; --it->second; + ++new_elements_size; } else if (select_query->distinct || hasArrayJoin(elem)) { /// ARRAY JOIN cannot be optimized out since it may change number of rows, /// so as DISTINCT. - new_elements.push_back(elem); + new_elements[result_index] = elem; + ++new_elements_size; } else { @@ -508,13 +510,20 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const /// Never remove untuple. It's result column may be in required columns. /// It is not easy to analyze untuple here, because types were not calculated yet. if (func && func->name == "untuple") - new_elements.push_back(elem); - + { + new_elements[result_index] = elem; + ++new_elements_size; + } /// removing aggregation can change number of rows, so `count()` result in outer sub-query would be wrong if (func && AggregateUtils::isAggregateFunction(*func) && !select_query->groupBy()) - new_elements.push_back(elem); + { + new_elements[result_index] = elem; + ++new_elements_size; + } } } + /// Remove empty nodes. + std::erase(new_elements, ASTPtr{}); if (select_query->interpolate()) { @@ -834,17 +843,43 @@ std::vector getWindowFunctions(ASTPtr & query, const ASTSel class MarkTupleLiteralsAsLegacyData { public: - using TypeToVisit = ASTLiteral; + struct Data + { + }; - static void visit(ASTLiteral & literal, ASTPtr &) + static void visitLiteral(ASTLiteral & literal, ASTPtr &) { if (literal.value.getType() == Field::Types::Tuple) literal.use_legacy_column_name_of_tuple = true; } + static void visitFunction(ASTFunction & func, ASTPtr &ast) + { + if (func.name == "tuple" && func.arguments && !func.arguments->children.empty()) + { + // re-write tuple() function as literal + if (auto literal = func.toLiteral()) + { + ast = literal; + visitLiteral(*typeid_cast(ast.get()), ast); + } + } + } + + static void visit(ASTPtr & ast, Data &) + { + if (auto * identifier = typeid_cast(ast.get())) + visitFunction(*identifier, ast); + if (auto * identifier = typeid_cast(ast.get())) + visitLiteral(*identifier, ast); + } + + static bool needChildVisit(const ASTPtr & /*parent*/, const ASTPtr & /*child*/) + { + return true; + } }; -using MarkTupleLiteralsAsLegacyMatcher = OneTypeMatcher; -using MarkTupleLiteralsAsLegacyVisitor = InDepthNodeVisitor; +using MarkTupleLiteralsAsLegacyVisitor = InDepthNodeVisitor; void markTupleLiteralsAsLegacy(ASTPtr & query) { @@ -1167,7 +1202,6 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( size_t subquery_depth = select_options.subquery_depth; bool remove_duplicates = select_options.remove_duplicates; - bool reorder_columns_as_required_header = select_options.reorder_columns_as_required_header; const auto & settings = getContext()->getSettingsRef(); @@ -1219,7 +1253,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside. /// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost) /// and before 'executeScalarSubqueries', 'analyzeAggregation', etc. to avoid excessive calculations. - removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates, reorder_columns_as_required_header); + removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates); /// Executing scalar subqueries - replacing them with constant values. executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 24649128cee..4b328f0466e 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -42,18 +42,19 @@ #include #include #include -#include #include +#include +#include #include #include #include -#include #include +#include #include -#include -#include -#include #include +#include +#include +#include #include #include @@ -68,6 +69,7 @@ #include #include +#include #include @@ -416,7 +418,9 @@ static std::tuple executeQueryImpl( chassert(txn->getState() != MergeTreeTransaction::COMMITTING); chassert(txn->getState() != MergeTreeTransaction::COMMITTED); if (txn->getState() == MergeTreeTransaction::ROLLED_BACK && !ast->as() && !ast->as()) - throw Exception(ErrorCodes::INVALID_TRANSACTION, "Cannot execute query because current transaction failed. Expecting ROLLBACK statement."); + throw Exception( + ErrorCodes::INVALID_TRANSACTION, + "Cannot execute query because current transaction failed. Expecting ROLLBACK statement"); } /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter), @@ -498,6 +502,7 @@ static std::tuple executeQueryImpl( BlockIO res; String query_for_logging; + std::shared_ptr implicit_txn_control{}; try { @@ -621,11 +626,37 @@ static std::tuple executeQueryImpl( if (!table_id.empty()) context->setInsertionTable(table_id); - if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (context->getCurrentTransaction() && settings.throw_on_unsupported_query_inside_transaction) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Async inserts inside transactions are not supported"); + if (settings.implicit_transaction && settings.throw_on_unsupported_query_inside_transaction) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Async inserts with 'implicit_transaction' are not supported"); } else { + /// We need to start the (implicit) transaction before getting the interpreter as this will get links to the latest snapshots + if (!context->getCurrentTransaction() && settings.implicit_transaction && !ast->as()) + { + try + { + if (context->isGlobalContext()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot create transactions"); + + /// If there is no session (which is the default for the HTTP Handler), set up one just for this as it is necessary + /// to control the transaction lifetime + if (!context->hasSessionContext()) + context->makeSessionContext(); + + auto tc = std::make_shared(ast, context); + tc->executeBegin(context->getSessionContext()); + implicit_txn_control = std::move(tc); + } + catch (Exception & e) + { + e.addMessage("while starting a transaction with 'implicit_transaction'"); + throw; + } + } + interpreter = InterpreterFactory::get(ast, context, SelectQueryOptions(stage).setInternal(internal)); if (context->getCurrentTransaction() && !interpreter->supportsTransactions() && @@ -813,15 +844,16 @@ static std::tuple executeQueryImpl( }; /// Also make possible for caller to log successful query finish and exception during execution. - auto finish_callback = [elem, context, ast, - log_queries, - log_queries_min_type = settings.log_queries_min_type, - log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), - log_processors_profiles = settings.log_processors_profiles, - status_info_to_query_log, - pulling_pipeline = pipeline.pulling() - ] - (QueryPipeline & query_pipeline) mutable + auto finish_callback = [elem, + context, + ast, + log_queries, + log_queries_min_type = settings.log_queries_min_type, + log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), + log_processors_profiles = settings.log_processors_profiles, + status_info_to_query_log, + implicit_txn_control, + pulling_pipeline = pipeline.pulling()](QueryPipeline & query_pipeline) mutable { QueryStatus * process_list_elem = context->getProcessListElement(); @@ -942,15 +974,40 @@ static std::tuple executeQueryImpl( opentelemetry_span_log->add(span); } + + if (implicit_txn_control) + { + try + { + implicit_txn_control->executeCommit(context->getSessionContext()); + implicit_txn_control.reset(); + } + catch (const Exception &) + { + /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception + /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN + implicit_txn_control.reset(); + throw; + } + } }; - auto exception_callback = [elem, context, ast, - log_queries, - log_queries_min_type = settings.log_queries_min_type, - log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), - quota(quota), status_info_to_query_log] () mutable + auto exception_callback = [elem, + context, + ast, + log_queries, + log_queries_min_type = settings.log_queries_min_type, + log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), + quota(quota), + status_info_to_query_log, + implicit_txn_control]() mutable { - if (auto txn = context->getCurrentTransaction()) + if (implicit_txn_control) + { + implicit_txn_control->executeRollback(context->getSessionContext()); + implicit_txn_control.reset(); + } + else if (auto txn = context->getCurrentTransaction()) txn->onException(); if (quota) @@ -1000,7 +1057,6 @@ static std::tuple executeQueryImpl( { ProfileEvents::increment(ProfileEvents::FailedInsertQuery); } - }; res.finish_callback = std::move(finish_callback); @@ -1009,8 +1065,15 @@ static std::tuple executeQueryImpl( } catch (...) { - if (auto txn = context->getCurrentTransaction()) + if (implicit_txn_control) + { + implicit_txn_control->executeRollback(context->getSessionContext()); + implicit_txn_control.reset(); + } + else if (auto txn = context->getCurrentTransaction()) + { txn->onException(); + } if (!internal) { diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index de920eaddbf..15d4f7929f8 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -18,7 +19,6 @@ #include #include -#include #include #include @@ -27,6 +27,12 @@ namespace fs = std::filesystem; namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; +} + static void executeCreateQuery( const String & query, ContextMutablePtr context, @@ -71,12 +77,6 @@ static void loadDatabase( ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } - else if (fs::exists(fs::path(database_path))) - { - /// TODO Remove this code (it's required for compatibility with versions older than 20.7) - /// Database exists, but .sql file is absent. It's old-style Ordinary database (e.g. system or default) - database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database) + " ENGINE = Ordinary"; - } else { /// It's first server run and we need create default and system databases. @@ -95,6 +95,15 @@ static void loadDatabase( } } +static void checkUnsupportedVersion(ContextMutablePtr context, const String & database_name) +{ + /// Produce better exception message + String metadata_path = context->getPath() + "metadata/" + database_name; + if (fs::exists(fs::path(metadata_path))) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Data directory for {} database exists, but metadata file does not. " + "Probably you are trying to upgrade from version older than 20.7. " + "If so, you should upgrade through intermediate version.", database_name); +} void loadMetadata(ContextMutablePtr context, const String & default_database_name) { @@ -118,43 +127,33 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam if (it->is_symlink()) continue; - const auto current_file = it->path().filename().string(); - if (!it->is_directory()) - { - /// TODO: DETACH DATABASE PERMANENTLY ? - if (fs::path(current_file).extension() == ".sql") - { - String db_name = fs::path(current_file).stem(); - if (!isSystemOrInformationSchema(db_name)) - databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name); - } - - /// Temporary fails may be left from previous server runs. - if (fs::path(current_file).extension() == ".tmp") - { - LOG_WARNING(log, "Removing temporary file {}", it->path().string()); - try - { - fs::remove(it->path()); - } - catch (...) - { - /// It does not prevent server to startup. - tryLogCurrentException(log); - } - } - + if (it->is_directory()) continue; + + const auto current_file = it->path().filename().string(); + + /// TODO: DETACH DATABASE PERMANENTLY ? + if (fs::path(current_file).extension() == ".sql") + { + String db_name = fs::path(current_file).stem(); + if (!isSystemOrInformationSchema(db_name)) + databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name); } - /// For '.svn', '.gitignore' directory and similar. - if (current_file.at(0) == '.') - continue; - - if (isSystemOrInformationSchema(current_file)) - continue; - - databases.emplace(unescapeForFileName(current_file), it->path().string()); + /// Temporary fails may be left from previous server runs. + if (fs::path(current_file).extension() == ".tmp") + { + LOG_WARNING(log, "Removing temporary file {}", it->path().string()); + try + { + fs::remove(it->path()); + } + catch (...) + { + /// It does not prevent server to startup. + tryLogCurrentException(log); + } + } } /// clickhouse-local creates DatabaseMemory as default database by itself @@ -162,7 +161,10 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam bool create_default_db_if_not_exists = !default_database_name.empty(); bool metadata_dir_for_default_db_already_exists = databases.contains(default_database_name); if (create_default_db_if_not_exists && !metadata_dir_for_default_db_already_exists) + { + checkUnsupportedVersion(context, default_database_name); databases.emplace(default_database_name, std::filesystem::path(path) / escapeForFileName(default_database_name)); + } TablesLoader::Databases loaded_databases; for (const auto & [name, db_path] : databases) @@ -192,13 +194,14 @@ static void loadSystemDatabaseImpl(ContextMutablePtr context, const String & dat { String path = context->getPath() + "metadata/" + database_name; String metadata_file = path + ".sql"; - if (fs::exists(fs::path(path)) || fs::exists(fs::path(metadata_file))) + if (fs::exists(fs::path(metadata_file))) { /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. loadDatabase(context, database_name, path, true); } else { + checkUnsupportedVersion(context, database_name); /// Initialize system database manually String database_create_query = "CREATE DATABASE "; database_create_query += database_name; @@ -208,6 +211,122 @@ static void loadSystemDatabaseImpl(ContextMutablePtr context, const String & dat } } +static void convertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const DatabasePtr & database) +{ + /// It's kind of C++ script that creates temporary database with Atomic engine, + /// moves all tables to it, drops old database and then renames new one to old name. + + Poco::Logger * log = &Poco::Logger::get("loadMetadata"); + + String name = database->getDatabaseName(); + + String tmp_name = fmt::format(".tmp_convert.{}.{}", name, thread_local_rng()); + + String name_quoted = backQuoteIfNeed(name); + String tmp_name_quoted = backQuoteIfNeed(tmp_name); + + LOG_INFO(log, "Will convert database {} from Ordinary to Atomic", name_quoted); + + String create_database_query = fmt::format("CREATE DATABASE IF NOT EXISTS {}", tmp_name_quoted); + auto res = executeQuery(create_database_query, context, true); + executeTrivialBlockIO(res, context); + res = {}; + auto tmp_database = DatabaseCatalog::instance().getDatabase(tmp_name); + assert(tmp_database->getEngineName() == "Atomic"); + + size_t num_tables = 0; + for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) + { + ++num_tables; + auto id = iterator->table()->getStorageID(); + id.database_name = tmp_name; + iterator->table()->checkTableCanBeRenamed(id); + } + + LOG_INFO(log, "Will move {} tables to {}", num_tables, tmp_name_quoted); + + for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) + { + auto id = iterator->table()->getStorageID(); + String qualified_quoted_name = id.getFullTableName(); + id.database_name = tmp_name; + String tmp_qualified_quoted_name = id.getFullTableName(); + + String move_table_query = fmt::format("RENAME TABLE {} TO {}", qualified_quoted_name, tmp_qualified_quoted_name); + res = executeQuery(move_table_query, context, true); + executeTrivialBlockIO(res, context); + res = {}; + } + + LOG_INFO(log, "Moved all tables from {} to {}", name_quoted, tmp_name_quoted); + + if (!database->empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Database {} is not empty after moving tables", name_quoted); + + String drop_query = fmt::format("DROP DATABASE {}", name_quoted); + res = executeQuery(drop_query, context, true); + executeTrivialBlockIO(res, context); + res = {}; + + String rename_query = fmt::format("RENAME DATABASE {} TO {}", tmp_name_quoted, name_quoted); + res = executeQuery(rename_query, context, true); + executeTrivialBlockIO(res, context); + + LOG_INFO(log, "Finished database engine conversion of {}", name_quoted); +} + +void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const DatabasePtr & database) +{ + if (database->getEngineName() != "Ordinary") + return; + + if (context->getSettingsRef().allow_deprecated_database_ordinary) + return; + + try + { + /// It's not quite correct to run DDL queries while database is not started up. + startupSystemTables(); + + auto local_context = Context::createCopy(context); + local_context->setSetting("check_table_dependencies", false); + convertOrdinaryDatabaseToAtomic(local_context, database); + + auto new_database = DatabaseCatalog::instance().getDatabase(DatabaseCatalog::SYSTEM_DATABASE); + UUID db_uuid = new_database->getUUID(); + std::vector tables_uuids; + for (auto iterator = new_database->getTablesIterator(context); iterator->isValid(); iterator->next()) + tables_uuids.push_back(iterator->uuid()); + + /// Reload database just in case (and update logger name) + String detach_query = fmt::format("DETACH DATABASE {}", backQuoteIfNeed(DatabaseCatalog::SYSTEM_DATABASE)); + auto res = executeQuery(detach_query, context, true); + executeTrivialBlockIO(res, context); + res = {}; + + /// Unlock UUID mapping, because it will be locked again on database reload. + /// It's safe to do during metadata loading, because cleanup task is not started yet. + DatabaseCatalog::instance().removeUUIDMappingFinally(db_uuid); + for (const auto & uuid : tables_uuids) + DatabaseCatalog::instance().removeUUIDMappingFinally(uuid); + + loadSystemDatabaseImpl(context, DatabaseCatalog::SYSTEM_DATABASE, "Atomic"); + TablesLoader::Databases databases = + { + {DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::instance().getSystemDatabase()}, + }; + TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true}; + loader.loadTables(); + + /// Will startup tables usual way + } + catch (Exception & e) + { + e.addMessage("While trying to convert {} to Atomic", database->getDatabaseName()); + throw; + } +} + void startupSystemTables() { diff --git a/src/Interpreters/loadMetadata.h b/src/Interpreters/loadMetadata.h index e918b5f530c..8dc332defc5 100644 --- a/src/Interpreters/loadMetadata.h +++ b/src/Interpreters/loadMetadata.h @@ -19,4 +19,8 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam /// so we startup system tables after all databases are loaded. void startupSystemTables(); +/// Converts database with Ordinary engine to Atomic. Does nothing if database is not Ordinary. +/// Can be called only during server startup when there are no queries from users. +void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const DatabasePtr & database); + } diff --git a/src/Parsers/ASTCreateIndexQuery.cpp b/src/Parsers/ASTCreateIndexQuery.cpp new file mode 100644 index 00000000000..50470fbc1e4 --- /dev/null +++ b/src/Parsers/ASTCreateIndexQuery.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** Get the text that identifies this element. */ +String ASTCreateIndexQuery::getID(char delim) const +{ + return "CreateIndexQuery" + (delim + getDatabase()) + delim + getTable(); +} + +ASTPtr ASTCreateIndexQuery::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + + res->index_name = index_name->clone(); + res->children.push_back(res->index_name); + + res->index_decl = index_decl->clone(); + res->children.push_back(res->index_decl); + + cloneTableOptions(*res); + + return res; +} + +void ASTCreateIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + frame.need_parens = false; + + std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + + settings.ostr << "CREATE INDEX " << (if_not_exists ? "IF NOT EXISTS " : ""); + index_name->formatImpl(settings, state, frame); + settings.ostr << " ON "; + + settings.ostr << (settings.hilite ? hilite_none : ""); + + if (table) + { + if (database) + { + settings.ostr << indent_str << backQuoteIfNeed(getDatabase()); + settings.ostr << "."; + } + settings.ostr << indent_str << backQuoteIfNeed(getTable()); + } + + formatOnCluster(settings); + + if (!cluster.empty()) + settings.ostr << " "; + + index_decl->formatImpl(settings, state, frame); +} + +ASTPtr ASTCreateIndexQuery::convertToASTAlterCommand() const +{ + auto command = std::make_shared(); + command->type = ASTAlterCommand::ADD_INDEX; + command->index = index_name->clone(); + command->index_decl = index_decl->clone(); + command->if_not_exists = if_not_exists; + + return command; +} + +} diff --git a/src/Parsers/ASTCreateIndexQuery.h b/src/Parsers/ASTCreateIndexQuery.h new file mode 100644 index 00000000000..424a0e493d9 --- /dev/null +++ b/src/Parsers/ASTCreateIndexQuery.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +/** CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value + */ + +class ASTCreateIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster +{ +public: + ASTPtr index_name; + + /// Stores the IndexDeclaration here. + ASTPtr index_decl; + + bool if_not_exists{false}; + + String getID(char delim) const override; + + ASTPtr clone() const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override + { + return removeOnCluster(clone(), params.default_database); + } + + QueryKind getQueryKind() const override { return QueryKind::Create; } + + /// Convert ASTCreateIndexQuery to ASTAlterCommand + ASTPtr convertToASTAlterCommand() const; + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/src/Parsers/ASTDescribeCacheQuery.h b/src/Parsers/ASTDescribeCacheQuery.h new file mode 100644 index 00000000000..17ec697d565 --- /dev/null +++ b/src/Parsers/ASTDescribeCacheQuery.h @@ -0,0 +1,28 @@ +#pragma once +#include + +namespace DB +{ + +class ASTDescribeCacheQuery : public ASTQueryWithOutput +{ +public: + String cache_name; + + String getID(char) const override { return "DescribeCacheQuery"; } + + ASTPtr clone() const override + { + auto res = std::make_shared(*this); + cloneOutputOptions(*res); + return res; + } + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << "DESCRIBE CACHE" << (settings.hilite ? hilite_none : "") << " " << cache_name; + } +}; + +} diff --git a/src/Parsers/ASTDropIndexQuery.cpp b/src/Parsers/ASTDropIndexQuery.cpp new file mode 100644 index 00000000000..a07336a2d26 --- /dev/null +++ b/src/Parsers/ASTDropIndexQuery.cpp @@ -0,0 +1,65 @@ +#include +#include +#include + + +namespace DB +{ + +/** Get the text that identifies this element. */ +String ASTDropIndexQuery::getID(char delim) const +{ + return "CreateIndexQuery" + (delim + getDatabase()) + delim + getTable(); +} + +ASTPtr ASTDropIndexQuery::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + + res->index_name = index_name->clone(); + res->children.push_back(res->index_name); + + cloneTableOptions(*res); + + return res; +} + +void ASTDropIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + frame.need_parens = false; + + std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + + settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : ""); + index_name->formatImpl(settings, state, frame); + settings.ostr << " ON "; + + settings.ostr << (settings.hilite ? hilite_none : ""); + + if (table) + { + if (database) + { + settings.ostr << indent_str << backQuoteIfNeed(getDatabase()); + settings.ostr << "."; + } + settings.ostr << indent_str << backQuoteIfNeed(getTable()); + } + + formatOnCluster(settings); +} + +ASTPtr ASTDropIndexQuery::convertToASTAlterCommand() const +{ + auto command = std::make_shared(); + command->type = ASTAlterCommand::DROP_INDEX; + command->index = index_name->clone(); + command->if_exists = if_exists; + + return command; +} + +} diff --git a/src/Parsers/ASTDropIndexQuery.h b/src/Parsers/ASTDropIndexQuery.h new file mode 100644 index 00000000000..6c2aaeb5936 --- /dev/null +++ b/src/Parsers/ASTDropIndexQuery.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** DROP INDEX [IF EXISTS] name on [db].name + */ + +class ASTDropIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster +{ +public: + + ASTPtr index_name; + + bool if_exists{false}; + + String getID(char delim) const override; + + ASTPtr clone() const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override + { + return removeOnCluster(clone(), params.default_database); + } + + QueryKind getQueryKind() const override { return QueryKind::Drop; } + + /// Convert ASTDropIndexQuery to ASTAlterCommand + ASTPtr convertToASTAlterCommand() const; + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index 9e815ee75de..11c1dd4c47a 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -72,8 +72,8 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState if (permanently) settings.ostr << " PERMANENTLY"; - if (no_delay) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " NO DELAY" << (settings.hilite ? hilite_none : ""); + if (sync) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " SYNC" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index ef2b609fbac..b4c96353a09 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -31,7 +31,7 @@ public: /// Same as above bool is_view{false}; - bool no_delay{false}; + bool sync{false}; // We detach the object permanently, so it will not be reattached back during server restart. bool permanently{false}; diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index d8ebf825674..d223661451e 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -25,9 +25,19 @@ ASTPtr ASTIndexDeclaration::clone() const void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - s.ostr << backQuoteIfNeed(name); - s.ostr << " "; - expr->formatImpl(s, state, frame); + if (part_of_create_index_query) + { + s.ostr << "("; + expr->formatImpl(s, state, frame); + s.ostr << ")"; + } + else + { + s.ostr << backQuoteIfNeed(name); + s.ostr << " "; + expr->formatImpl(s, state, frame); + } + s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); type->formatImpl(s, state, frame); s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index 8416ec6b0a6..e22c1da4489 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -16,6 +16,7 @@ public: IAST * expr; ASTFunction * type; UInt64 granularity; + bool part_of_create_index_query = false; /** Get the text that identifies this element. */ String getID(char) const override { return "Index"; } diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index 61d68c4a273..42cc0077efa 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -55,6 +55,12 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTER" << (settings.hilite ? hilite_none : ""); settings.ostr << " " << backQuoteIfNeed(cluster_str); } + else if (caches) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CACHES" << (settings.hilite ? hilite_none : ""); + formatLike(settings); + formatLimit(settings, state, frame); + } else if (m_settings) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW " << (changed ? "CHANGED " : "") << "SETTINGS" << diff --git a/src/Parsers/ASTShowTablesQuery.h b/src/Parsers/ASTShowTablesQuery.h index 57383dff66f..04cf9d6645a 100644 --- a/src/Parsers/ASTShowTablesQuery.h +++ b/src/Parsers/ASTShowTablesQuery.h @@ -9,7 +9,7 @@ namespace DB { -/** Query SHOW TABLES or SHOW DATABASES or SHOW CLUSTERS +/** Query SHOW TABLES or SHOW DATABASES or SHOW CLUSTERS or SHOW CACHES */ class ASTShowTablesQuery : public ASTQueryWithOutput { @@ -21,6 +21,7 @@ public: bool m_settings{false}; bool changed{false}; bool temporary{false}; + bool caches{false}; String cluster_str; String from; diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index 9a3c1dd8bed..bc3af03a3c4 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -840,7 +840,6 @@ bool ParserAlterCommandList::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; } - bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { auto query = std::make_shared(); diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp new file mode 100644 index 00000000000..af0d9064626 --- /dev/null +++ b/src/Parsers/ParserCreateIndexQuery.cpp @@ -0,0 +1,123 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_type("TYPE"); + ParserKeyword s_granularity("GRANULARITY"); + + ParserDataType data_type_p; + ParserExpression expression_p; + ParserUnsignedInteger granularity_p; + + ASTPtr expr; + ASTPtr type; + ASTPtr granularity; + + /// Skip name parser for SQL-standard CREATE INDEX + if (!expression_p.parse(pos, expr, expected)) + return false; + + if (!s_type.ignore(pos, expected)) + return false; + + if (!data_type_p.parse(pos, type, expected)) + return false; + + if (!s_granularity.ignore(pos, expected)) + return false; + + if (!granularity_p.parse(pos, granularity, expected)) + return false; + + auto index = std::make_shared(); + index->part_of_create_index_query = true; + index->granularity = granularity->as().value.safeGet(); + index->set(index->expr, expr); + index->set(index->type, type); + node = index; + + return true; +} + +bool ParserCreateIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + auto query = std::make_shared(); + node = query; + + ParserKeyword s_create("CREATE"); + ParserKeyword s_index("INDEX"); + ParserKeyword s_if_not_exists("IF NOT EXISTS"); + ParserKeyword s_on("ON"); + ParserIdentifier index_name_p; + ParserCreateIndexDeclaration parser_create_idx_decl; + + ASTPtr index_name; + ASTPtr index_decl; + + String cluster_str; + bool if_not_exists = false; + + if (!s_create.ignore(pos, expected)) + return false; + + if (!s_index.ignore(pos, expected)) + return false; + + if (s_if_not_exists.ignore(pos, expected)) + if_not_exists = true; + + if (!index_name_p.parse(pos, index_name, expected)) + return false; + + /// ON [db.] table_name + if (!s_on.ignore(pos, expected)) + return false; + + if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) + return false; + + /// [ON cluster_name] + if (s_on.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + + if (!parser_create_idx_decl.parse(pos, index_decl, expected)) + return false; + + auto & ast_index_decl = index_decl->as(); + ast_index_decl.name = index_name->as().name(); + + query->index_name = index_name; + query->children.push_back(index_name); + + query->index_decl = index_decl; + query->children.push_back(index_decl); + + query->if_not_exists = if_not_exists; + query->cluster = cluster_str; + + if (query->database) + query->children.push_back(query->database); + + if (query->table) + query->children.push_back(query->table); + + return true; +} + +} diff --git a/src/Parsers/ParserCreateIndexQuery.h b/src/Parsers/ParserCreateIndexQuery.h new file mode 100644 index 00000000000..3cb91cd03c6 --- /dev/null +++ b/src/Parsers/ParserCreateIndexQuery.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace DB +{ + +/** Query like this: + * CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value + */ + +class ParserCreateIndexQuery : public IParserBase +{ +protected: + const char * getName() const override { return "CREATE INDEX query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + + +/** Parser for index declaration in create index, where name is ignored. */ +class ParserCreateIndexDeclaration : public IParserBase +{ +public: + ParserCreateIndexDeclaration() = default; + +protected: + const char * getName() const override { return "index declaration in create index"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index e4a3f87f288..4b6ab67e22f 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -467,6 +467,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe ParserKeyword s_from("FROM"); ParserKeyword s_on("ON"); ParserToken s_dot(TokenType::Dot); + ParserToken s_comma(TokenType::Comma); ParserToken s_lparen(TokenType::OpeningRoundBracket); ParserToken s_rparen(TokenType::ClosingRoundBracket); ParserStorage storage_p; @@ -574,6 +575,10 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (!table_properties_p.parse(pos, columns_list, expected)) return false; + /// We allow a trailing comma in the columns list for user convenience. + /// Although it diverges from the SQL standard slightly. + s_comma.ignore(pos, expected); + if (!s_rparen.ignore(pos, expected)) return false; diff --git a/src/Parsers/ParserDescribeCacheQuery.cpp b/src/Parsers/ParserDescribeCacheQuery.cpp new file mode 100644 index 00000000000..e64c1a850f2 --- /dev/null +++ b/src/Parsers/ParserDescribeCacheQuery.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include + +namespace DB +{ + + +bool ParserDescribeCacheQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword p_describe("DESCRIBE"); + ParserKeyword p_desc("DESC"); + ParserKeyword p_cache("CACHE"); + ParserLiteral p_cache_name; + + if ((!p_describe.ignore(pos, expected) && !p_desc.ignore(pos, expected)) + || !p_cache.ignore(pos, expected)) + return false; + + auto query = std::make_shared(); + + ASTPtr ast; + if (!p_cache_name.parse(pos, ast, expected)) + return false; + + query->cache_name = ast->as()->value.safeGet(); + node = query; + + return true; +} + + +} diff --git a/src/Parsers/ParserDescribeCacheQuery.h b/src/Parsers/ParserDescribeCacheQuery.h new file mode 100644 index 00000000000..ae717875678 --- /dev/null +++ b/src/Parsers/ParserDescribeCacheQuery.h @@ -0,0 +1,20 @@ +#pragma once + + +#include +#include + + +namespace DB +{ + +/** Query (DESCRIBE | DESC) CACHE 'cache_name' + */ +class ParserDescribeCacheQuery : public IParserBase +{ +protected: + const char * getName() const override { return "DESCRIBE CACHE query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ParserDropIndexQuery.cpp b/src/Parsers/ParserDropIndexQuery.cpp new file mode 100644 index 00000000000..89ed4f01838 --- /dev/null +++ b/src/Parsers/ParserDropIndexQuery.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserDropIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + auto query = std::make_shared(); + node = query; + + ParserKeyword s_drop("DROP"); + ParserKeyword s_index("INDEX"); + ParserKeyword s_on("ON"); + ParserKeyword s_if_exists("IF EXISTS"); + ParserIdentifier index_name_p; + + String cluster_str; + bool if_exists = false; + + if (!s_drop.ignore(pos, expected)) + return false; + + if (!s_index.ignore(pos, expected)) + return false; + + if (s_if_exists.ignore(pos, expected)) + if_exists = true; + + if (!index_name_p.parse(pos, query->index_name, expected)) + return false; + + /// ON [db.] table_name + if (!s_on.ignore(pos, expected)) + return false; + + if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) + return false; + + /// [ON cluster_name] + if (s_on.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + + query->cluster = std::move(cluster_str); + } + + if (query->index_name) + query->children.push_back(query->index_name); + + query->if_exists = if_exists; + + if (query->database) + query->children.push_back(query->database); + + if (query->table) + query->children.push_back(query->table); + + return true; +} + +} diff --git a/src/Parsers/ParserDropIndexQuery.h b/src/Parsers/ParserDropIndexQuery.h new file mode 100644 index 00000000000..fc25ace469b --- /dev/null +++ b/src/Parsers/ParserDropIndexQuery.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace DB +{ + +/** Query like this: + * DROP INDEX [IF EXISTS] name ON [db].name + */ + +class ParserDropIndexQuery : public IParserBase +{ +protected: + const char * getName() const override { return "DROP INDEX query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ParserDropQuery.cpp b/src/Parsers/ParserDropQuery.cpp index e7263aa47f5..f40a39e6b2f 100644 --- a/src/Parsers/ParserDropQuery.cpp +++ b/src/Parsers/ParserDropQuery.cpp @@ -31,7 +31,7 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons bool temporary = false; bool is_dictionary = false; bool is_view = false; - bool no_delay = false; + bool sync = false; bool permanently = false; if (s_database.ignore(pos, expected)) @@ -83,7 +83,7 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons /// actually for TRUNCATE NO DELAY / SYNC means nothing if (s_no_delay.ignore(pos, expected) || s_sync.ignore(pos, expected)) - no_delay = true; + sync = true; auto query = std::make_shared(); node = query; @@ -93,7 +93,7 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons query->temporary = temporary; query->is_dictionary = is_dictionary; query->is_view = is_view; - query->no_delay = no_delay; + query->sync = sync; query->permanently = permanently; query->database = database; query->table = table; diff --git a/src/Parsers/ParserQuery.cpp b/src/Parsers/ParserQuery.cpp index eaea5dd0f5f..a3cafee65d7 100644 --- a/src/Parsers/ParserQuery.cpp +++ b/src/Parsers/ParserQuery.cpp @@ -2,7 +2,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -43,6 +45,8 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserCreateSettingsProfileQuery create_settings_profile_p; ParserCreateFunctionQuery create_function_p; ParserDropFunctionQuery drop_function_p; + ParserCreateIndexQuery create_index_p; + ParserDropIndexQuery drop_index_p; ParserDropAccessEntityQuery drop_access_entity_p; ParserGrantQuery grant_p; ParserSetRoleQuery set_role_p; @@ -63,6 +67,8 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) || create_settings_profile_p.parse(pos, node, expected) || create_function_p.parse(pos, node, expected) || drop_function_p.parse(pos, node, expected) + || create_index_p.parse(pos, node, expected) + || drop_index_p.parse(pos, node, expected) || drop_access_entity_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected) || external_ddl_p.parse(pos, node, expected) diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index 6041f986a49..a6af19fc670 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserSelectWithUnionQuery select_p; ParserTablePropertiesQuery table_p; ParserDescribeTableQuery describe_table_p; + ParserDescribeCacheQuery describe_cache_p; ParserShowProcesslistQuery show_processlist_p; ParserCreateQuery create_p; ParserAlterQuery alter_p; @@ -59,6 +61,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || show_create_access_entity_p.parse(pos, query, expected) /// should be before `show_tables_p` || show_tables_p.parse(pos, query, expected) || table_p.parse(pos, query, expected) + || describe_cache_p.parse(pos, query, expected) || describe_table_p.parse(pos, query, expected) || show_processlist_p.parse(pos, query, expected) || create_p.parse(pos, query, expected) diff --git a/src/Parsers/ParserShowTablesQuery.cpp b/src/Parsers/ParserShowTablesQuery.cpp index b5c8687321e..46c783b8271 100644 --- a/src/Parsers/ParserShowTablesQuery.cpp +++ b/src/Parsers/ParserShowTablesQuery.cpp @@ -24,6 +24,7 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserKeyword s_clusters("CLUSTERS"); ParserKeyword s_cluster("CLUSTER"); ParserKeyword s_dictionaries("DICTIONARIES"); + ParserKeyword s_caches("CACHES"); ParserKeyword s_settings("SETTINGS"); ParserKeyword s_changed("CHANGED"); ParserKeyword s_from("FROM"); @@ -91,6 +92,10 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec return false; } } + else if (s_caches.ignore(pos, expected)) + { + query->caches = true; + } else if (s_cluster.ignore(pos, expected)) { query->cluster = true; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 34279bdd300..7854f7a2400 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -360,6 +360,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & ASTPtr ast; if (path_parser.parse(pos, ast, expected)) res->filesystem_cache_path = ast->as()->value.safeGet(); + parseQueryWithOnCluster(res, pos, expected); break; } diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index c792d828e44..f922ff048b2 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -602,8 +602,8 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & Columns columns_list; UInt64 num_rows = name_to_column_ptr.begin()->second->length(); - columns_list.reserve(header.rows()); - std::unordered_map nested_tables; + columns_list.reserve(header.columns()); + std::unordered_map>> nested_tables; bool skipped = false; for (size_t column_i = 0, columns = header.columns(); column_i < columns; ++column_i) { @@ -613,55 +613,57 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & if (case_insensitive_matching) boost::to_lower(search_column_name); - bool read_from_nested = false; - String nested_table_name = Nested::extractTableName(header_column.name); - String search_nested_table_name = nested_table_name; - if (case_insensitive_matching) - boost::to_lower(search_nested_table_name); - + ColumnWithTypeAndName column; if (!name_to_column_ptr.contains(search_column_name)) { + bool read_from_nested = false; /// Check if it's a column from nested table. - if (import_nested && name_to_column_ptr.contains(search_nested_table_name)) + if (import_nested) { - if (!nested_tables.contains(search_nested_table_name)) + String nested_table_name = Nested::extractTableName(header_column.name); + String search_nested_table_name = nested_table_name; + if (case_insensitive_matching) + boost::to_lower(search_nested_table_name); + if (name_to_column_ptr.contains(search_nested_table_name)) { - std::shared_ptr arrow_column = name_to_column_ptr[search_nested_table_name]; - ColumnsWithTypeAndName cols - = {readColumnFromArrowColumn(arrow_column, nested_table_name, format_name, false, dictionary_values, true, true, false, skipped)}; - Block block(cols); - nested_tables[search_nested_table_name] = std::make_shared(Nested::flatten(block)); + if (!nested_tables.contains(search_nested_table_name)) + { + std::shared_ptr arrow_column = name_to_column_ptr[search_nested_table_name]; + ColumnsWithTypeAndName cols = {readColumnFromArrowColumn( + arrow_column, nested_table_name, format_name, false, dictionary_values, true, true, false, skipped)}; + BlockPtr block_ptr = std::make_shared(cols); + auto column_extractor = std::make_shared(*block_ptr, case_insensitive_matching); + nested_tables[search_nested_table_name] = {block_ptr, column_extractor}; + } + auto nested_column = nested_tables[search_nested_table_name].second->extractColumn(search_column_name); + if (nested_column) + { + column = *nested_column; + if (case_insensitive_matching) + column.name = header_column.name; + read_from_nested = true; + } } - - read_from_nested = nested_tables[search_nested_table_name]->has(header_column.name, case_insensitive_matching); } - if (!read_from_nested) { if (!allow_missing_columns) throw Exception{ErrorCodes::THERE_IS_NO_COLUMN, "Column '{}' is not presented in input data.", header_column.name}; - - ColumnWithTypeAndName column; - column.name = header_column.name; - column.type = header_column.type; - column.column = header_column.column->cloneResized(num_rows); - columns_list.push_back(std::move(column.column)); - continue; + else + { + column.name = header_column.name; + column.type = header_column.type; + column.column = header_column.column->cloneResized(num_rows); + columns_list.push_back(std::move(column.column)); + continue; + } } } - - - ColumnWithTypeAndName column; - if (read_from_nested) - { - column = nested_tables[search_nested_table_name]->getByName(header_column.name, case_insensitive_matching); - if (case_insensitive_matching) - column.name = header_column.name; - } else { auto arrow_column = name_to_column_ptr[search_column_name]; - column = readColumnFromArrowColumn(arrow_column, header_column.name, format_name, false, dictionary_values, true, true, false, skipped); + column = readColumnFromArrowColumn( + arrow_column, header_column.name, format_name, false, dictionary_values, true, true, false, skipped); } try @@ -689,23 +691,17 @@ std::vector ArrowColumnToCHColumn::getMissingColumns(const arrow::Schema { std::vector missing_columns; auto block_from_arrow = arrowSchemaToCHHeader(schema, format_name, false, &header, case_insensitive_matching); - auto flatten_block_from_arrow = Nested::flatten(block_from_arrow); + NestedColumnExtractHelper nested_columns_extractor(block_from_arrow, case_insensitive_matching); for (size_t i = 0, columns = header.columns(); i < columns; ++i) { const auto & header_column = header.getByPosition(i); - bool read_from_nested = false; - String nested_table_name = Nested::extractTableName(header_column.name); if (!block_from_arrow.has(header_column.name, case_insensitive_matching)) { - if (import_nested && block_from_arrow.has(nested_table_name, case_insensitive_matching)) - read_from_nested = flatten_block_from_arrow.has(header_column.name, case_insensitive_matching); - - if (!read_from_nested) + if (!import_nested || !nested_columns_extractor.extractColumn(header_column.name)) { if (!allow_missing_columns) throw Exception{ErrorCodes::THERE_IS_NO_COLUMN, "Column '{}' is not presented in input data.", header_column.name}; - missing_columns.push_back(i); } } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp index 34b9305c0e1..32ab391cf8c 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp @@ -20,7 +20,7 @@ namespace DB } { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (background_exception) std::rethrow_exception(background_exception); @@ -30,7 +30,7 @@ namespace DB void ParallelFormattingOutputFormat::addChunk(Chunk chunk, ProcessingUnitType type, bool can_throw_exception) { { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (background_exception && can_throw_exception) std::rethrow_exception(background_exception); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index edde79be896..fb58f5765c1 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -236,7 +236,7 @@ private: void onBackgroundException() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!background_exception) { background_exception = std::current_exception(); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index d3e167d35c6..318bcaed466 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -121,7 +121,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa void ParallelParsingInputFormat::onBackgroundException(size_t offset) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!background_exception) { background_exception = std::current_exception(); @@ -233,7 +233,7 @@ Chunk ParallelParsingInputFormat::generate() else { // Pass the unit back to the segmentator. - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); unit.status = READY_TO_INSERT; segmentator_condvar.notify_all(); } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 28f821d6f3f..0a4b12084eb 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -19,13 +19,13 @@ namespace DB { -static ITransformingStep::Traits getTraits() +static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number) { return ITransformingStep::Traits { { .preserves_distinct_columns = false, /// Actually, we may check that distinct names are in aggregation keys - .returns_single_stream = true, + .returns_single_stream = should_produce_results_in_order_of_bucket_number, /// Actually, may also return single stream if should_produce_results_in_order_of_bucket_number = false .preserves_number_of_streams = false, .preserves_sorting = false, }, @@ -69,19 +69,19 @@ AggregatingStep::AggregatingStep( Aggregator::Params params_, GroupingSetsParamsList grouping_sets_params_, bool final_, - bool only_merge_, size_t max_block_size_, size_t aggregation_in_order_max_block_bytes_, size_t merge_threads_, size_t temporary_data_merge_threads_, bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, - SortDescription group_by_sort_description_) - : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_), getTraits(), false) + SortDescription group_by_sort_description_, + bool should_produce_results_in_order_of_bucket_number_) + : ITransformingStep( + input_stream_, appendGroupingColumn(params_.getHeader(input_stream_.header, final_), grouping_sets_params_), getTraits(should_produce_results_in_order_of_bucket_number_), false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(final_) - , only_merge(only_merge_) , max_block_size(max_block_size_) , aggregation_in_order_max_block_bytes(aggregation_in_order_max_block_bytes_) , merge_threads(merge_threads_) @@ -89,6 +89,7 @@ AggregatingStep::AggregatingStep( , storage_has_evenly_distributed_read(storage_has_evenly_distributed_read_) , group_by_info(std::move(group_by_info_)) , group_by_sort_description(std::move(group_by_sort_description_)) + , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) { } @@ -121,7 +122,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B * 1. Parallel aggregation is done, and the results should be merged in parallel. * 2. An aggregation is done with store of temporary data on the disk, and they need to be merged in a memory efficient way. */ - auto transform_params = std::make_shared(std::move(params), final, only_merge); + const auto src_header = pipeline.getHeader(); + auto transform_params = std::make_shared(src_header, std::move(params), final); if (!grouping_sets_params.empty()) { @@ -153,7 +155,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B { Aggregator::Params params_for_set { - transform_params->params.src_header, grouping_sets_params[i].used_keys, transform_params->params.aggregates, transform_params->params.overflow_row, @@ -168,10 +169,9 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B transform_params->params.min_free_disk_space, transform_params->params.compile_aggregate_expressions, transform_params->params.min_count_to_compile_aggregate_expression, - transform_params->params.intermediate_header, - transform_params->params.stats_collecting_params - }; - auto transform_params_for_set = std::make_shared(std::move(params_for_set), final, only_merge); + /* only_merge */ false, + transform_params->params.stats_collecting_params}; + auto transform_params_for_set = std::make_shared(src_header, std::move(params_for_set), final); if (streams > 1) { @@ -234,15 +234,15 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B grouping_node = &dag->materializeNode(*grouping_node); index.push_back(grouping_node); - size_t missign_column_index = 0; const auto & missing_columns = grouping_sets_params[set_counter].missing_keys; for (size_t i = 0; i < output_header.columns(); ++i) { auto & col = output_header.getByPosition(i); - if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) + const auto it = std::find_if( + missing_columns.begin(), missing_columns.end(), [&](const auto & missing_col) { return missing_col == col.name; }); + if (it != missing_columns.end()) { - ++missign_column_index; auto column_with_default = col.column->cloneEmpty(); col.type->insertDefaultInto(*column_with_default); auto column = ColumnConst::create(std::move(column_with_default), 0); @@ -353,7 +353,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); }); - pipeline.resize(1); + /// We add the explicit resize here, but not in case of aggregating in order, since AIO don't use two-level hash tables and thus returns only buckets with bucket_number = -1. + pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : pipeline.getNumStreams(), true /* force */); aggregating = collector.detachProcessors(0); } @@ -391,4 +392,12 @@ void AggregatingStep::describePipeline(FormatSettings & settings) const } } +void AggregatingStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), + appendGroupingColumn(params.getHeader(input_streams.front().header, final), grouping_sets_params), + getDataStreamTraits()); +} + } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 1be74da583a..0e982d76940 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -11,13 +11,10 @@ struct GroupingSetsParams { GroupingSetsParams() = default; - GroupingSetsParams(ColumnNumbers used_keys_, ColumnNumbers missing_keys_) - : used_keys(std::move(used_keys_)) - , missing_keys(std::move(missing_keys_)) - {} + GroupingSetsParams(Names used_keys_, Names missing_keys_) : used_keys(std::move(used_keys_)), missing_keys(std::move(missing_keys_)) { } - ColumnNumbers used_keys; - ColumnNumbers missing_keys; + Names used_keys; + Names missing_keys; }; using GroupingSetsParamsList = std::vector; @@ -33,14 +30,14 @@ public: Aggregator::Params params_, GroupingSetsParamsList grouping_sets_params_, bool final_, - bool only_merge_, size_t max_block_size_, size_t aggregation_in_order_max_block_bytes_, size_t merge_threads_, size_t temporary_data_merge_threads_, bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, - SortDescription group_by_sort_description_); + SortDescription group_by_sort_description_, + bool should_produce_results_in_order_of_bucket_number_); String getName() const override { return "Aggregating"; } @@ -54,10 +51,11 @@ public: const Aggregator::Params & getParams() const { return params; } private: + void updateOutputStream() override; + Aggregator::Params params; GroupingSetsParamsList grouping_sets_params; bool final; - bool only_merge; size_t max_block_size; size_t aggregation_in_order_max_block_bytes; size_t merge_threads; @@ -68,6 +66,10 @@ private: InputOrderInfoPtr group_by_info; SortDescription group_by_sort_description; + /// It determines if we should resize pipeline to 1 at the end. + /// Needed in case of distributed memory efficient aggregation. + const bool should_produce_results_in_order_of_bucket_number; + Processors aggregating_in_order; Processors aggregating_sorted; Processors finalizing; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 3ca5b9109e6..bd1908a4a6d 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -34,40 +34,19 @@ ArrayJoinStep::ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPt { } -void ArrayJoinStep::updateInputStream(DataStream input_stream, Block result_header) +void ArrayJoinStep::updateOutputStream() { output_stream = createOutputStream( - input_stream, - ArrayJoinTransform::transformHeader(input_stream.header, array_join), - getDataStreamTraits()); - - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); - res_header = std::move(result_header); + input_streams.front(), ArrayJoinTransform::transformHeader(input_streams.front().header, array_join), getDataStreamTraits()); } -void ArrayJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) +void ArrayJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) { bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; return std::make_shared(header, array_join, on_totals); }); - - if (res_header && !blocksHaveEqualStructure(res_header, output_stream->header)) - { - auto actions_dag = ActionsDAG::makeConvertingActions( - pipeline.getHeader().getColumnsWithTypeAndName(), - res_header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name); - - auto actions = std::make_shared(actions_dag, settings.getActionsSettings()); - - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, actions); - }); - } } void ArrayJoinStep::describeActions(FormatSettings & settings) const diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index 83df4d021e8..2d9b2ebd0c8 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -13,18 +13,17 @@ public: explicit ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPtr array_join_); String getName() const override { return "ArrayJoin"; } - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; - void updateInputStream(DataStream input_stream, Block result_header); - const ArrayJoinActionPtr & arrayJoin() const { return array_join; } private: + void updateOutputStream() override; + ArrayJoinActionPtr array_join; - Block res_header; }; } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 6b6f9d361ef..94d841ff095 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -49,6 +49,11 @@ void CreatingSetStep::transformPipeline(QueryPipelineBuilder & pipeline, const B pipeline.addCreatingSetsTransform(getOutputStream().header, std::move(subquery_for_set), network_transfer_limits, getContext()); } +void CreatingSetStep::updateOutputStream() +{ + output_stream = createOutputStream(input_streams.front(), Block{}, getDataStreamTraits()); +} + void CreatingSetStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index e20c28e10f4..20cdd24c8a9 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -27,6 +27,8 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override; + String description; SubqueryForSet subquery_for_set; SizeLimits network_transfer_limits; diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp index 91c85a08412..b0c57491085 100644 --- a/src/Processors/QueryPlan/CubeStep.cpp +++ b/src/Processors/QueryPlan/CubeStep.cpp @@ -24,14 +24,15 @@ static ITransformingStep::Traits getTraits() }; } -CubeStep::CubeStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) - , keys_size(params_->params.keys_size) +CubeStep::CubeStep(const DataStream & input_stream_, Aggregator::Params params_, bool final_) + : ITransformingStep(input_stream_, appendGroupingSetColumn(params_.getHeader(input_stream_.header, final_)), getTraits()) + , keys_size(params_.keys_size) , params(std::move(params_)) + , final(final_) { /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number) @@ -59,13 +60,23 @@ void CubeStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQue if (stream_type == QueryPipelineBuilder::StreamType::Totals) return addGroupingSetForTotals(header, settings, (UInt64(1) << keys_size) - 1); - return std::make_shared(header, std::move(params)); + auto transform_params = std::make_shared(header, std::move(params), final); + return std::make_shared(header, std::move(transform_params)); }); } const Aggregator::Params & CubeStep::getParams() const { - return params->params; + return params; } +void CubeStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), appendGroupingSetColumn(params.getHeader(input_streams.front().header, final)), getDataStreamTraits()); + + /// Aggregation keys are distinct + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); +} } diff --git a/src/Processors/QueryPlan/CubeStep.h b/src/Processors/QueryPlan/CubeStep.h index d3e26f9379f..87f22de7fc6 100644 --- a/src/Processors/QueryPlan/CubeStep.h +++ b/src/Processors/QueryPlan/CubeStep.h @@ -13,7 +13,7 @@ using AggregatingTransformParamsPtr = std::shared_ptrdistinct_columns.empty() /// Columns already distinct, do nothing + && (!pre_distinct /// Main distinct + || input_streams.front().has_single_port)) /// pre_distinct for single port works as usual one + { + /// Build distinct set. + for (const auto & name : columns) + output_stream->distinct_columns.insert(name); + } +} + } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index a48a779425d..a4424e01d72 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -24,6 +24,8 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override; + SizeLimits set_size_limits; UInt64 limit_hint; Names columns; diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index 33d2ad6e1cf..c9f1e1ce242 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -38,19 +38,6 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, ActionsDAGPtr a updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } -void ExpressionStep::updateInputStream(DataStream input_stream, bool keep_header) -{ - Block out_header = keep_header ? std::move(output_stream->header) - : ExpressionTransform::transformHeader(input_stream.header, *actions_dag); - output_stream = createOutputStream( - input_stream, - std::move(out_header), - getDataStreamTraits()); - - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); -} - void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { auto expression = std::make_shared(actions_dag, settings.getActionsSettings()); @@ -101,4 +88,10 @@ void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Expression", expression->toTree()); } +void ExpressionStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), ExpressionTransform::transformHeader(input_streams.front().header, *actions_dag), getDataStreamTraits()); +} + } diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h index 94c2ba21bc1..96869a9e9ca 100644 --- a/src/Processors/QueryPlan/ExpressionStep.h +++ b/src/Processors/QueryPlan/ExpressionStep.h @@ -20,8 +20,6 @@ public: void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; - void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(FormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } @@ -29,6 +27,8 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; private: + void updateOutputStream() override; + ActionsDAGPtr actions_dag; }; diff --git a/src/Processors/QueryPlan/ExtremesStep.h b/src/Processors/QueryPlan/ExtremesStep.h index 7898796306c..57ccef077aa 100644 --- a/src/Processors/QueryPlan/ExtremesStep.h +++ b/src/Processors/QueryPlan/ExtremesStep.h @@ -12,6 +12,12 @@ public: String getName() const override { return "Extremes"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + +private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } }; } diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index 41bc2f44dba..8a370786820 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -57,4 +57,12 @@ void FillingStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Sort Description", explainSortDescription(sort_description)); } +void FillingStep::updateOutputStream() +{ + if (!input_streams.front().has_single_port) + throw Exception("FillingStep expects single input", ErrorCodes::LOGICAL_ERROR); + + output_stream = createOutputStream( + input_streams.front(), FillingTransform::transformHeader(input_streams.front().header, sort_description), getDataStreamTraits()); +} } diff --git a/src/Processors/QueryPlan/FillingStep.h b/src/Processors/QueryPlan/FillingStep.h index 0393b2c525b..4e1b5b3654d 100644 --- a/src/Processors/QueryPlan/FillingStep.h +++ b/src/Processors/QueryPlan/FillingStep.h @@ -22,6 +22,8 @@ public: const SortDescription & getSortDescription() const { return sort_description; } private: + void updateOutputStream() override; + SortDescription sort_description; InterpolateDescriptionPtr interpolate_description; }; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index df75c37dc97..ff58abf8874 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -46,25 +46,6 @@ FilterStep::FilterStep( updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } -void FilterStep::updateInputStream(DataStream input_stream, bool keep_header) -{ - Block out_header = std::move(output_stream->header); - if (keep_header) - out_header = FilterTransform::transformHeader( - input_stream.header, - *actions_dag, - filter_column_name, - remove_filter_column); - - output_stream = createOutputStream( - input_stream, - std::move(out_header), - getDataStreamTraits()); - - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); -} - void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { auto expression = std::make_shared(actions_dag, settings.getActionsSettings()); @@ -124,4 +105,13 @@ void FilterStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Expression", expression->toTree()); } +void FilterStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), + FilterTransform::transformHeader(input_streams.front().header, *actions_dag, filter_column_name, remove_filter_column), + getDataStreamTraits()); +} + + } diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h index 7ac5bc036e0..daea1e13c58 100644 --- a/src/Processors/QueryPlan/FilterStep.h +++ b/src/Processors/QueryPlan/FilterStep.h @@ -20,8 +20,6 @@ public: String getName() const override { return "Filter"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; - void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; @@ -30,6 +28,8 @@ public: bool removesFilterColumn() const { return remove_filter_column; } private: + void updateOutputStream() override; + ActionsDAGPtr actions_dag; String filter_column_name; bool remove_filter_column; diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index 8f3641dd5bd..008642c71ee 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -55,6 +55,19 @@ public: const TransformTraits & getTransformTraits() const { return transform_traits; } const DataStreamTraits & getDataStreamTraits() const { return data_stream_traits; } + /// Updates the input stream of the given step. Used during query plan optimizations. + /// It won't do any validation of a new stream, so it is your responsibility to ensure that this update doesn't break anything + /// (e.g. you update data stream traits or correctly remove / add columns). + void updateInputStream(DataStream input_stream) + { + input_streams.clear(); + input_streams.emplace_back(std::move(input_stream)); + + updateOutputStream(); + + updateDistinctColumns(output_stream->header, output_stream->distinct_columns); + } + void describePipeline(FormatSettings & settings) const override; /// Append extra processors for this step. @@ -73,6 +86,8 @@ protected: TransformTraits transform_traits; private: + virtual void updateOutputStream() = 0; + /// We collect processors got after pipeline transformation. Processors processors; bool collect_processors; diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 983be9d45fb..209d91af4d8 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -40,6 +40,14 @@ void JoinStep::describePipeline(FormatSettings & settings) const IQueryPlanStep::describePipeline(processors, settings); } +void JoinStep::updateLeftStream(const DataStream & left_stream_) +{ + input_streams = {left_stream_, input_streams.at(1)}; + output_stream = DataStream{ + .header = JoiningTransform::transformHeader(left_stream_.header, join), + }; +} + static ITransformingStep::Traits getStorageJoinTraits() { return ITransformingStep::Traits @@ -87,4 +95,11 @@ void FilledJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu }); } +void FilledJoinStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), JoiningTransform::transformHeader(input_streams.front().header, join), getDataStreamTraits()); +} + + } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index b9d3dff1b65..17a0cc2ae63 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -28,6 +28,8 @@ public: const JoinPtr & getJoin() const { return join; } + void updateLeftStream(const DataStream & left_stream_); + private: JoinPtr join; size_t max_block_size; @@ -47,6 +49,8 @@ public: void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; private: + void updateOutputStream() override; + JoinPtr join; size_t max_block_size; }; diff --git a/src/Processors/QueryPlan/LimitByStep.h b/src/Processors/QueryPlan/LimitByStep.h index eb91be8a814..0edda3247d6 100644 --- a/src/Processors/QueryPlan/LimitByStep.h +++ b/src/Processors/QueryPlan/LimitByStep.h @@ -20,6 +20,11 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } + size_t group_length; size_t group_offset; Names columns; diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index aff7472e4aa..144ac16f0d5 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -36,13 +36,6 @@ LimitStep::LimitStep( { } -void LimitStep::updateInputStream(DataStream input_stream) -{ - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); - output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); -} - void LimitStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { auto transform = std::make_shared( diff --git a/src/Processors/QueryPlan/LimitStep.h b/src/Processors/QueryPlan/LimitStep.h index f5bceeb29c7..1ae6b73cc3d 100644 --- a/src/Processors/QueryPlan/LimitStep.h +++ b/src/Processors/QueryPlan/LimitStep.h @@ -31,12 +31,14 @@ public: return limit + offset; } - /// Change input stream when limit is pushed up. TODO: add clone() for steps. - void updateInputStream(DataStream input_stream); - bool withTies() const { return with_ties; } private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } + size_t limit; size_t offset; bool always_read_till_end; diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 8dfb9f9c923..d74a6174f00 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -7,13 +7,13 @@ namespace DB { -static ITransformingStep::Traits getTraits() +static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number) { return ITransformingStep::Traits { { .preserves_distinct_columns = false, - .returns_single_stream = true, + .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, }, @@ -25,23 +25,29 @@ static ITransformingStep::Traits getTraits() MergingAggregatedStep::MergingAggregatedStep( const DataStream & input_stream_, - AggregatingTransformParamsPtr params_, + Aggregator::Params params_, + bool final_, bool memory_efficient_aggregation_, size_t max_threads_, - size_t memory_efficient_merge_threads_) - : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) - , params(params_) + size_t memory_efficient_merge_threads_, + bool should_produce_results_in_order_of_bucket_number_) + : ITransformingStep( + input_stream_, params_.getHeader(input_stream_.header, final_), getTraits(should_produce_results_in_order_of_bucket_number_)) + , params(std::move(params_)) + , final(final_) , memory_efficient_aggregation(memory_efficient_aggregation_) , max_threads(max_threads_) , memory_efficient_merge_threads(memory_efficient_merge_threads_) + , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) { /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.intermediate_header.getByPosition(key).name); + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); } void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { + auto transform_params = std::make_shared(pipeline.getHeader(), std::move(params), final); if (!memory_efficient_aggregation) { /// We union several sources into one, paralleling the work. @@ -49,9 +55,7 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c /// Now merge the aggregated blocks pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, params, max_threads); - }); + { return std::make_shared(header, transform_params, max_threads); }); } else { @@ -59,18 +63,30 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c ? static_cast(memory_efficient_merge_threads) : static_cast(max_threads); - pipeline.addMergingAggregatedMemoryEfficientTransform(params, num_merge_threads); + pipeline.addMergingAggregatedMemoryEfficientTransform(transform_params, num_merge_threads); } + + pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : max_threads); } void MergingAggregatedStep::describeActions(FormatSettings & settings) const { - return params->params.explain(settings.out, settings.offset); + return params.explain(settings.out, settings.offset); } void MergingAggregatedStep::describeActions(JSONBuilder::JSONMap & map) const { - params->params.explain(map); + params.explain(map); } +void MergingAggregatedStep::updateOutputStream() +{ + output_stream = createOutputStream(input_streams.front(), params.getHeader(input_streams.front().header, final), getDataStreamTraits()); + + /// Aggregation keys are distinct + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); +} + + } diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index eeead41b5f9..419b43615bd 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include @@ -14,10 +15,12 @@ class MergingAggregatedStep : public ITransformingStep public: MergingAggregatedStep( const DataStream & input_stream_, - AggregatingTransformParamsPtr params_, + Aggregator::Params params_, + bool final_, bool memory_efficient_aggregation_, size_t max_threads_, - size_t memory_efficient_merge_threads_); + size_t memory_efficient_merge_threads_, + bool should_produce_results_in_order_of_bucket_number_); String getName() const override { return "MergingAggregated"; } @@ -27,10 +30,17 @@ public: void describeActions(FormatSettings & settings) const override; private: - AggregatingTransformParamsPtr params; + void updateOutputStream() override; + + Aggregator::Params params; + bool final; bool memory_efficient_aggregation; size_t max_threads; size_t memory_efficient_merge_threads; + + /// It determines if we should resize pipeline to 1 at the end. + /// Needed in case of distributed memory efficient aggregation over distributed table. + const bool should_produce_results_in_order_of_bucket_number; }; } diff --git a/src/Processors/QueryPlan/OffsetStep.h b/src/Processors/QueryPlan/OffsetStep.h index f16559bcfad..a32835b62a6 100644 --- a/src/Processors/QueryPlan/OffsetStep.h +++ b/src/Processors/QueryPlan/OffsetStep.h @@ -19,6 +19,11 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } + size_t offset; }; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 2625bf38bf7..0c17c27e7aa 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -27,10 +27,19 @@ namespace DB::ErrorCodes namespace DB::QueryPlanOptimizations { -static size_t tryAddNewFilterStep( - QueryPlan::Node * parent_node, - QueryPlan::Nodes & nodes, - const Names & allowed_inputs) +static bool filterColumnIsNotAmongAggregatesArguments(const AggregateDescriptions & aggregates, const std::string & filter_column_name) +{ + for (const auto & aggregate : aggregates) + { + const auto & argument_names = aggregate.argument_names; + if (std::find(argument_names.begin(), argument_names.end(), filter_column_name) != argument_names.end()) + return false; + } + return true; +} + +static size_t +tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const Names & allowed_inputs, bool can_remove_filter = true) { QueryPlan::Node * child_node = parent_node->children.front(); @@ -62,11 +71,6 @@ static size_t tryAddNewFilterStep( /// Filter column was replaced to constant. const bool filter_is_constant = filter_node && filter_node->column && isColumnConst(*filter_node->column); - if (!filter_node || filter_is_constant) - /// This means that all predicates of filter were pushed down. - /// Replace current actions to expression, as we don't need to filter anything. - parent = std::make_unique(child->getOutputStream(), expression); - /// Add new Filter step before Aggregating. /// Expression/Filter -> Aggregating -> Something auto & node = nodes.emplace_back(); @@ -77,21 +81,31 @@ static size_t tryAddNewFilterStep( /// New filter column is the first one. auto split_filter_column_name = (*split_filter->getIndex().begin())->result_name; node.step = std::make_unique( - node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), true); + node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), can_remove_filter); + + if (auto * transforming_step = dynamic_cast(child.get())) + { + transforming_step->updateInputStream(node.step->getOutputStream()); + } + else + { + if (auto * join = typeid_cast(child.get())) + join->updateLeftStream(node.step->getOutputStream()); + else + throw Exception( + ErrorCodes::LOGICAL_ERROR, "We are trying to push down a filter through a step for which we cannot update input stream"); + } + + if (!filter_node || filter_is_constant) + /// This means that all predicates of filter were pushed down. + /// Replace current actions to expression, as we don't need to filter anything. + parent = std::make_unique(child->getOutputStream(), expression); + else + filter->updateInputStream(child->getOutputStream()); return 3; } -static Names getAggregatingKeys(const Aggregator::Params & params) -{ - Names keys; - keys.reserve(params.keys.size()); - for (auto pos : params.keys) - keys.push_back(params.src_header.getByPosition(pos).name); - - return keys; -} - size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) { if (parent_node->children.size() != 1) @@ -112,9 +126,14 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * aggregating = typeid_cast(child.get())) { const auto & params = aggregating->getParams(); - Names keys = getAggregatingKeys(params); + const auto & keys = params.keys; - if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, keys)) + const bool filter_column_is_not_among_aggregation_keys + = std::find(keys.begin(), keys.end(), filter->getFilterColumnName()) == keys.end(); + const bool can_remove_filter = filter_column_is_not_among_aggregation_keys + && filterColumnIsNotAmongAggregatesArguments(params.aggregates, filter->getFilterColumnName()); + + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, keys, can_remove_filter)) return updated_steps; } @@ -213,7 +232,9 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes allowed_keys.push_back(name); } - if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys)) + const bool can_remove_filter + = std::find(source_columns.begin(), source_columns.end(), filter->getFilterColumnName()) == source_columns.end(); + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys, can_remove_filter)) return updated_steps; } } diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index e20c5f93d6e..36aab41df49 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -36,30 +36,6 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node auto description = parent->getStepDescription(); - /// All actions was moved before ARRAY JOIN. Swap Expression and ArrayJoin. - if (split_actions.second->trivial()) - { - auto expected_header = parent->getOutputStream().header; - - /// Expression/Filter -> ArrayJoin - std::swap(parent, child); - /// ArrayJoin -> Expression/Filter - - if (expression_step) - child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), - std::move(split_actions.first)); - else - child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), - std::move(split_actions.first), - filter_step->getFilterColumnName(), - filter_step->removesFilterColumn()); - - child->setStepDescription(std::move(description)); - - array_join_step->updateInputStream(child->getOutputStream(), expected_header); - return 2; - } - /// Add new expression step before ARRAY JOIN. /// Expression/Filter -> ArrayJoin -> Something auto & node = nodes.emplace_back(); @@ -70,7 +46,7 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node node.step = std::make_unique(node.children.at(0)->step->getOutputStream(), std::move(split_actions.first)); node.step->setStepDescription(description); - array_join_step->updateInputStream(node.step->getOutputStream(), {}); + array_join_step->updateInputStream(node.step->getOutputStream()); if (expression_step) parent = std::make_unique(array_join_step->getOutputStream(), split_actions.second); diff --git a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp index 872adfeccc1..c3b03a5385f 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp @@ -70,8 +70,6 @@ size_t tryExecuteFunctionsAfterSorting(QueryPlan::Node * parent_node, QueryPlan: // so far the origin Expression (parent_node) -> Sorting (child_node) -> NeededCalculations (node_with_needed) sorting_step->updateInputStream(getChildOutputStream(*child_node)); - auto input_header = sorting_step->getInputStreams().at(0).header; - sorting_step->updateOutputStream(std::move(input_header)); auto description = parent_step->getStepDescription(); parent_step = std::make_unique(child_step->getOutputStream(), std::move(unneeded_for_sorting)); diff --git a/src/Processors/QueryPlan/RollupStep.cpp b/src/Processors/QueryPlan/RollupStep.cpp index 3b061f9c246..169976195ea 100644 --- a/src/Processors/QueryPlan/RollupStep.cpp +++ b/src/Processors/QueryPlan/RollupStep.cpp @@ -22,14 +22,15 @@ static ITransformingStep::Traits getTraits() }; } -RollupStep::RollupStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) +RollupStep::RollupStep(const DataStream & input_stream_, Aggregator::Params params_, bool final_) + : ITransformingStep(input_stream_, appendGroupingSetColumn(params_.getHeader(input_stream_.header, final_)), getTraits()) , params(std::move(params_)) - , keys_size(params->params.keys_size) + , keys_size(params.keys_size) + , final(final_) { /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number); @@ -43,8 +44,20 @@ void RollupStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ if (stream_type == QueryPipelineBuilder::StreamType::Totals) return addGroupingSetForTotals(header, settings, keys_size); - return std::make_shared(header, std::move(params)); + auto transform_params = std::make_shared(header, std::move(params), true); + return std::make_shared(header, std::move(transform_params)); }); } +void RollupStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), appendGroupingSetColumn(params.getHeader(input_streams.front().header, final)), getDataStreamTraits()); + + /// Aggregation keys are distinct + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); +} + + } diff --git a/src/Processors/QueryPlan/RollupStep.h b/src/Processors/QueryPlan/RollupStep.h index 3dce6f74d9f..c59bf9f3ee9 100644 --- a/src/Processors/QueryPlan/RollupStep.h +++ b/src/Processors/QueryPlan/RollupStep.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include @@ -12,15 +13,18 @@ using AggregatingTransformParamsPtr = std::shared_ptrsort_mode = DataStream::SortMode::Stream; } -void SortingStep::updateInputStream(DataStream input_stream) +void SortingStep::updateOutputStream() { - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); -} - -void SortingStep::updateOutputStream(Block result_header) -{ - output_stream = createOutputStream(input_streams.at(0), std::move(result_header), getDataStreamTraits()); + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); output_stream->sort_description = result_description; output_stream->sort_mode = DataStream::SortMode::Stream; - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void SortingStep::updateLimit(size_t limit_) diff --git a/src/Processors/QueryPlan/SortingStep.h b/src/Processors/QueryPlan/SortingStep.h index 23a31b14093..ce78bb863bf 100644 --- a/src/Processors/QueryPlan/SortingStep.h +++ b/src/Processors/QueryPlan/SortingStep.h @@ -49,14 +49,12 @@ public: /// Add limit or change it to lower value. void updateLimit(size_t limit_); - void updateInputStream(DataStream input_stream); - void updateOutputStream(Block result_header); - SortDescription getSortDescription() const { return result_description; } void convertToFinishSorting(SortDescription prefix_description); private: + void updateOutputStream() override; enum class Type { diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 5e7e7011e0e..bb918a1a02d 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -27,7 +27,7 @@ static ITransformingStep::Traits getTraits(bool has_filter) TotalsHavingStep::TotalsHavingStep( const DataStream & input_stream_, - const ColumnsMask & aggregates_mask_, + const AggregateDescriptions & aggregates_, bool overflow_row_, const ActionsDAGPtr & actions_dag_, const std::string & filter_column_, @@ -36,16 +36,16 @@ TotalsHavingStep::TotalsHavingStep( double auto_include_threshold_, bool final_) : ITransformingStep( - input_stream_, - TotalsHavingTransform::transformHeader( - input_stream_.header, - actions_dag_.get(), - filter_column_, - remove_filter_, - final_, - aggregates_mask_), - getTraits(!filter_column_.empty())) - , aggregates_mask(aggregates_mask_) + input_stream_, + TotalsHavingTransform::transformHeader( + input_stream_.header, + actions_dag_.get(), + filter_column_, + remove_filter_, + final_, + getAggregatesMask(input_stream_.header, aggregates_)), + getTraits(!filter_column_.empty())) + , aggregates(aggregates_) , overflow_row(overflow_row_) , actions_dag(actions_dag_) , filter_column_name(filter_column_) @@ -62,7 +62,7 @@ void TotalsHavingStep::transformPipeline(QueryPipelineBuilder & pipeline, const auto totals_having = std::make_shared( pipeline.getHeader(), - aggregates_mask, + getAggregatesMask(pipeline.getHeader(), aggregates), overflow_row, expression_actions, filter_column_name, @@ -125,4 +125,19 @@ void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const } } +void TotalsHavingStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), + TotalsHavingTransform::transformHeader( + input_streams.front().header, + actions_dag.get(), + filter_column_name, + remove_filter, + final, + getAggregatesMask(input_streams.front().header, aggregates)), + getDataStreamTraits()); +} + + } diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h index 4ad741a1b44..b60eab78b53 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.h +++ b/src/Processors/QueryPlan/TotalsHavingStep.h @@ -15,15 +15,15 @@ class TotalsHavingStep : public ITransformingStep { public: TotalsHavingStep( - const DataStream & input_stream_, - const ColumnsMask & aggregates_mask_, - bool overflow_row_, - const ActionsDAGPtr & actions_dag_, - const std::string & filter_column_, - bool remove_filter_, - TotalsMode totals_mode_, - double auto_include_threshold_, - bool final_); + const DataStream & input_stream_, + const AggregateDescriptions & aggregates_, + bool overflow_row_, + const ActionsDAGPtr & actions_dag_, + const std::string & filter_column_, + bool remove_filter_, + TotalsMode totals_mode_, + double auto_include_threshold_, + bool final_); String getName() const override { return "TotalsHaving"; } @@ -35,7 +35,10 @@ public: const ActionsDAGPtr & getActions() const { return actions_dag; } private: - const ColumnsMask aggregates_mask; + void updateOutputStream() override; + + const AggregateDescriptions aggregates; + bool overflow_row; ActionsDAGPtr actions_dag; String filter_column_name; diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index 48d16ed321f..b67b394b57b 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -44,17 +44,13 @@ static Block addWindowFunctionResultColumns(const Block & block, return result; } -WindowStep::WindowStep(const DataStream & input_stream_, - const WindowDescription & window_description_, - const std::vector & window_functions_) - : ITransformingStep( - input_stream_, - addWindowFunctionResultColumns(input_stream_.header, - window_functions_), - getTraits()) +WindowStep::WindowStep( + const DataStream & input_stream_, + const WindowDescription & window_description_, + const std::vector & window_functions_) + : ITransformingStep(input_stream_, addWindowFunctionResultColumns(input_stream_.header, window_functions_), getTraits()) , window_description(window_description_) , window_functions(window_functions_) - , input_header(input_stream_.header) { // We don't remove any columns, only add, so probably we don't have to update // the output DataStream::distinct_columns. @@ -70,11 +66,12 @@ void WindowStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ // have resized it. pipeline.resize(1); - pipeline.addSimpleTransform([&](const Block & /*header*/) - { - return std::make_shared(input_header, - output_stream->header, window_description, window_functions); - }); + pipeline.addSimpleTransform( + [&](const Block & /*header*/) + { + return std::make_shared( + input_streams.front().header, output_stream->header, window_description, window_functions); + }); assertBlocksHaveEqualStructure(pipeline.getHeader(), output_stream->header, "WindowStep transform for '" + window_description.window_name + "'"); @@ -138,6 +135,14 @@ void WindowStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Functions", std::move(functions_array)); } +void WindowStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), addWindowFunctionResultColumns(input_streams.front().header, window_functions), getDataStreamTraits()); + + window_description.checkValid(); +} + const WindowDescription & WindowStep::getWindowDescription() const { return window_description; diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h index 9b58cceb972..0bafdcbe414 100644 --- a/src/Processors/QueryPlan/WindowStep.h +++ b/src/Processors/QueryPlan/WindowStep.h @@ -28,9 +28,10 @@ public: const WindowDescription & getWindowDescription() const; private: + void updateOutputStream() override; + WindowDescription window_description; std::vector window_functions; - Block input_header; }; } diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index bce54ace054..d8b022f0acb 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -15,28 +15,31 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( { current_key_value.resize(description.group_by_keys.size()); - ColumnNumbers keys; - for (const auto & key : description.group_by_keys) - keys.push_back(header.getPositionByName(key)); + const auto & keys = description.group_by_keys; key_columns.resize(description.group_by_keys.size()); AggregateDescriptions aggregates = description.aggregate_descriptions; - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - descr.arguments.push_back(header.getPositionByName(name)); - columns_for_aggregator.resize(description.aggregate_descriptions.size()); const Settings & settings = storage_.getContext()->getSettingsRef(); - Aggregator::Params params(header, keys, aggregates, - false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, 0, 0, - settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - storage_.getContext()->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); + Aggregator::Params params( + keys, + aggregates, + false, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + 0, + 0, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set, + storage_.getContext()->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); - aggregator = std::make_unique(params); + aggregator = std::make_unique(header, params); if (isMaxTTLExpired()) new_ttl_info.ttl_finished = true; diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index f435d46a066..ce50ae5eeee 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -87,7 +87,8 @@ void AggregatingInOrderTransform::consume(Chunk chunk) Columns key_columns(params->params.keys_size); for (size_t i = 0; i < params->params.keys_size; ++i) { - materialized_columns.push_back(columns.at(params->params.keys[i])->convertToFullColumnIfConst()); + const auto pos = inputs.front().getHeader().getPositionByName(params->params.keys[i]); + materialized_columns.push_back(chunk.getColumns().at(pos)->convertToFullColumnIfConst()); key_columns[i] = materialized_columns.back(); if (group_by_key) key_columns_raw[i] = materialized_columns.back().get(); @@ -95,7 +96,11 @@ void AggregatingInOrderTransform::consume(Chunk chunk) Aggregator::NestedColumnsHolder nested_columns_holder; Aggregator::AggregateFunctionInstructions aggregate_function_instructions; - params->aggregator.prepareAggregateInstructions(columns, aggregate_columns, materialized_columns, aggregate_function_instructions, nested_columns_holder); + if (!params->params.only_merge) + { + params->aggregator.prepareAggregateInstructions( + columns, aggregate_columns, materialized_columns, aggregate_function_instructions, nested_columns_holder); + } size_t key_end = 0; size_t key_begin = 0; @@ -123,7 +128,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) Int64 current_memory_usage = 0; Aggregator::AggregateColumnsConstData aggregate_columns_data(params->params.aggregates_size); - if (params->only_merge) + if (params->params.only_merge) { for (size_t i = 0, j = 0; i < columns.size(); ++i) { @@ -149,7 +154,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) /// Add data to aggr. state if interval is not empty. Empty when haven't found current key in new block. if (key_begin != key_end) { - if (params->only_merge) + if (params->params.only_merge) { if (group_by_key) params->aggregator.mergeOnBlockSmall(variants, key_begin, key_end, aggregate_columns_data, key_columns_raw); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index b5b254c3e3c..f8332742978 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -524,7 +524,7 @@ void AggregatingTransform::consume(Chunk chunk) src_rows += num_rows; src_bytes += chunk.bytes(); - if (params->only_merge) + if (params->params.only_merge) { auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); block = materializeBlock(block); @@ -549,7 +549,7 @@ void AggregatingTransform::initGenerate() /// To do this, we pass a block with zero rows to aggregate. if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) { - if (params->only_merge) + if (params->params.only_merge) params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys); else params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 8d62664da59..789fa970ebd 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -34,24 +34,21 @@ struct AggregatingTransformParams AggregatorListPtr aggregator_list_ptr; Aggregator & aggregator; bool final; - /// Merge data for aggregate projections. - bool only_merge = false; - AggregatingTransformParams(const Aggregator::Params & params_, bool final_, bool only_merge_) + AggregatingTransformParams(const Block & header, const Aggregator::Params & params_, bool final_) : params(params_) , aggregator_list_ptr(std::make_shared()) - , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), params)) + , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), header, params)) , final(final_) - , only_merge(only_merge_) { } - AggregatingTransformParams(const Aggregator::Params & params_, const AggregatorListPtr & aggregator_list_ptr_, bool final_, bool only_merge_) + AggregatingTransformParams( + const Block & header, const Aggregator::Params & params_, const AggregatorListPtr & aggregator_list_ptr_, bool final_) : params(params_) , aggregator_list_ptr(aggregator_list_ptr_) - , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), params)) + , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), header, params)) , final(final_) - , only_merge(only_merge_) { } diff --git a/src/Processors/Transforms/CubeTransform.cpp b/src/Processors/Transforms/CubeTransform.cpp index 83ed346dabe..b80ca29327f 100644 --- a/src/Processors/Transforms/CubeTransform.cpp +++ b/src/Processors/Transforms/CubeTransform.cpp @@ -12,9 +12,12 @@ namespace ErrorCodes CubeTransform::CubeTransform(Block header, AggregatingTransformParamsPtr params_) : IAccumulatingTransform(std::move(header), appendGroupingSetColumn(params_->getHeader())) , params(std::move(params_)) - , keys(params->params.keys) , aggregates_mask(getAggregatesMask(params->getHeader(), params->params.aggregates)) { + keys.reserve(params->params.keys_size); + for (const auto & key : params->params.keys) + keys.emplace_back(input.getHeader().getPositionByName(key)); + if (keys.size() >= 8 * sizeof(mask)) throw Exception("Too many keys are used for CubeTransform.", ErrorCodes::LOGICAL_ERROR); } diff --git a/src/Processors/Transforms/CubeTransform.h b/src/Processors/Transforms/CubeTransform.h index 4575a01935d..bd33eabd750 100644 --- a/src/Processors/Transforms/CubeTransform.h +++ b/src/Processors/Transforms/CubeTransform.h @@ -21,7 +21,7 @@ protected: private: AggregatingTransformParamsPtr params; - const ColumnNumbers keys; + ColumnNumbers keys; const ColumnsMask aggregates_mask; Chunks consumed_chunks; diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 13d039ebcae..73e89c774ae 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -13,10 +13,20 @@ DistinctSortedTransform::DistinctSortedTransform( : ISimpleTransform(header_, header_, true) , header(std::move(header_)) , description(std::move(sort_description)) - , columns_names(columns) + , column_names(columns) , limit_hint(limit_hint_) , set_size_limits(set_size_limits_) { + /// pre-calculate column positions to use during chunk transformation + const size_t num_columns = column_names.empty() ? header.columns() : column_names.size(); + column_positions.reserve(num_columns); + for (size_t i = 0; i < num_columns; ++i) + { + auto pos = column_names.empty() ? i : header.getPositionByName(column_names[i]); + const auto & col = header.getByPosition(pos).column; + if (col && !isColumnConst(*col)) + column_positions.emplace_back(pos); + } } void DistinctSortedTransform::transform(Chunk & chunk) @@ -119,24 +129,13 @@ bool DistinctSortedTransform::buildFilter( ColumnRawPtrs DistinctSortedTransform::getKeyColumns(const Chunk & chunk) const { - size_t columns = columns_names.empty() ? chunk.getNumColumns() : columns_names.size(); - ColumnRawPtrs column_ptrs; - column_ptrs.reserve(columns); - - for (size_t i = 0; i < columns; ++i) + column_ptrs.reserve(column_positions.size()); + for (const auto pos : column_positions) { - auto pos = i; - if (!columns_names.empty()) - pos = input.getHeader().getPositionByName(columns_names[i]); - const auto & column = chunk.getColumns()[pos]; - - /// Ignore all constant columns. - if (!isColumnConst(*column)) - column_ptrs.emplace_back(column.get()); + column_ptrs.emplace_back(column.get()); } - return column_ptrs; } diff --git a/src/Processors/Transforms/DistinctSortedTransform.h b/src/Processors/Transforms/DistinctSortedTransform.h index 0530a6689e9..72acde0716b 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.h +++ b/src/Processors/Transforms/DistinctSortedTransform.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -57,7 +58,8 @@ private: }; PreviousChunk prev_chunk; - Names columns_names; + Names column_names; + ColumnNumbers column_positions; ClearableSetVariants data; Sizes key_sizes; UInt64 limit_hint; diff --git a/src/Processors/Transforms/RollupTransform.cpp b/src/Processors/Transforms/RollupTransform.cpp index b69a691323c..e5351d1d5e2 100644 --- a/src/Processors/Transforms/RollupTransform.cpp +++ b/src/Processors/Transforms/RollupTransform.cpp @@ -8,9 +8,11 @@ namespace DB RollupTransform::RollupTransform(Block header, AggregatingTransformParamsPtr params_) : IAccumulatingTransform(std::move(header), appendGroupingSetColumn(params_->getHeader())) , params(std::move(params_)) - , keys(params->params.keys) , aggregates_mask(getAggregatesMask(params->getHeader(), params->params.aggregates)) { + keys.reserve(params->params.keys_size); + for (const auto & key : params->params.keys) + keys.emplace_back(input.getHeader().getPositionByName(key)); } void RollupTransform::consume(Chunk chunk) diff --git a/src/Processors/Transforms/RollupTransform.h b/src/Processors/Transforms/RollupTransform.h index 8fd27e3e6a2..1630df23579 100644 --- a/src/Processors/Transforms/RollupTransform.h +++ b/src/Processors/Transforms/RollupTransform.h @@ -20,7 +20,7 @@ protected: private: AggregatingTransformParamsPtr params; - const ColumnNumbers keys; + ColumnNumbers keys; const ColumnsMask aggregates_mask; Chunks consumed_chunks; diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 09805696472..3eb0f62cb01 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -28,7 +27,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; } // Interface for true window functions. It's not much of an interface, they just @@ -2202,109 +2200,6 @@ struct WindowFunctionNthValue final : public WindowFunction } }; -struct NonNegativeDerivativeState -{ - Float64 previous_metric = 0; - Float64 previous_timestamp = 0; -}; - -// nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL 1 SECOND]) -struct WindowFunctionNonNegativeDerivative final : public StatefulWindowFunction -{ - static constexpr size_t ARGUMENT_METRIC = 0; - static constexpr size_t ARGUMENT_TIMESTAMP = 1; - static constexpr size_t ARGUMENT_INTERVAL = 2; - - WindowFunctionNonNegativeDerivative(const std::string & name_, - const DataTypes & argument_types_, const Array & parameters_) - : StatefulWindowFunction(name_, argument_types_, parameters_) - { - if (!parameters.empty()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Function {} cannot be parameterized", name_); - } - - if (argument_types.size() != 2 && argument_types.size() != 3) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Function {} takes 2 or 3 arguments", name_); - } - - if (!isNumber(argument_types[ARGUMENT_METRIC])) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Argument {} must be a number, '{}' given", - ARGUMENT_METRIC, - argument_types[ARGUMENT_METRIC]->getName()); - } - - if (!isDateTime(argument_types[ARGUMENT_TIMESTAMP]) && !isDateTime64(argument_types[ARGUMENT_TIMESTAMP])) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Argument {} must be DateTime or DateTime64, '{}' given", - ARGUMENT_TIMESTAMP, - argument_types[ARGUMENT_TIMESTAMP]->getName()); - } - - if (argument_types.size() == 3) - { - const DataTypeInterval * interval_datatype = checkAndGetDataType(argument_types[ARGUMENT_INTERVAL].get()); - if (!interval_datatype) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Argument {} must be an INTERVAL, '{}' given", - ARGUMENT_INTERVAL, - argument_types[ARGUMENT_INTERVAL]->getName()); - } - if (!interval_datatype->getKind().isFixedLength()) - { - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The INTERVAL must be a week or shorter, '{}' given", - argument_types[ARGUMENT_INTERVAL]->getName()); - } - interval_length = interval_datatype->getKind().toAvgSeconds(); - interval_specified = true; - } - } - - - DataTypePtr getReturnType() const override { return argument_types[0]; } - - bool allocatesMemoryInArena() const override { return false; } - - void windowInsertResultInto(const WindowTransform * transform, - size_t function_index) override - { - const auto & current_block = transform->blockAt(transform->current_row); - const auto & workspace = transform->workspaces[function_index]; - auto & state = getState(workspace); - - auto interval_duration = interval_specified ? interval_length * - (*current_block.input_columns[workspace.argument_column_indices[ARGUMENT_INTERVAL]]).getFloat64(0) : 1; - - Float64 last_metric = state.previous_metric; - Float64 last_timestamp = state.previous_timestamp; - - Float64 curr_metric = WindowFunctionHelpers::getValue(transform, function_index, ARGUMENT_METRIC, transform->current_row); - Float64 curr_timestamp = WindowFunctionHelpers::getValue(transform, function_index, ARGUMENT_TIMESTAMP, transform->current_row); - - Float64 time_elapsed = curr_timestamp - last_timestamp; - Float64 metric_diff = curr_metric - last_metric; - Float64 result = (time_elapsed != 0) ? (metric_diff / time_elapsed * interval_duration) : 0; - - state.previous_metric = curr_metric; - state.previous_timestamp = curr_timestamp; - - WindowFunctionHelpers::setValueToOutputColumn(transform, function_index, result >= 0 ? result : 0); - } -private: - Float64 interval_length = 1; - bool interval_specified = false; -}; - void registerWindowFunctions(AggregateFunctionFactory & factory) { @@ -2404,13 +2299,6 @@ void registerWindowFunctions(AggregateFunctionFactory & factory) return std::make_shared( name, argument_types, parameters); }, properties}); - - factory.registerFunction("nonNegativeDerivative", {[](const std::string & name, - const DataTypes & argument_types, const Array & parameters, const Settings *) - { - return std::make_shared( - name, argument_types, parameters); - }, properties}); } } diff --git a/src/Server/HTTP/HTTPServerConnection.cpp b/src/Server/HTTP/HTTPServerConnection.cpp index 6b2ef32f6a4..92a994b3a4e 100644 --- a/src/Server/HTTP/HTTPServerConnection.cpp +++ b/src/Server/HTTP/HTTPServerConnection.cpp @@ -26,7 +26,7 @@ void HTTPServerConnection::run() { try { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!stopped && tcp_server.isOpen() && session.connected()) { HTTPServerResponse response(session); diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 2f7b03790ee..3ac9f50231d 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -10,7 +10,9 @@ #include #include #include - +#if USE_KRB5 +#include +#endif // USE_KRB5 namespace DB { @@ -18,8 +20,10 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NETWORK_ERROR; + #if USE_KRB5 extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int NO_ELEMENTS_IN_CONFIG; + extern const int KERBEROS_ERROR; + #endif // USE_KRB5 } const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs"; @@ -40,25 +44,28 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration String key_name; if (key == "hadoop_kerberos_keytab") { + #if USE_KRB5 need_kinit = true; hadoop_kerberos_keytab = config.getString(key_path); + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop_kerberos_keytab parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 continue; } else if (key == "hadoop_kerberos_principal") { + #if USE_KRB5 need_kinit = true; hadoop_kerberos_principal = config.getString(key_path); hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); - continue; - } - else if (key == "hadoop_kerberos_kinit_command") - { - need_kinit = true; - hadoop_kerberos_kinit_command = config.getString(key_path); + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop_kerberos_principal parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 continue; } else if (key == "hadoop_security_kerberos_ticket_cache_path") { + #if USE_KRB5 if (isUser) { throw Exception("hadoop.security.kerberos.ticket.cache.path cannot be set per user", @@ -67,6 +74,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration hadoop_security_kerberos_ticket_cache_path = config.getString(key_path); // standard param - pass further + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop.security.kerberos.ticket.cache.path parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 } key_name = boost::replace_all_copy(key, "_", "."); @@ -76,44 +86,21 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration } } -String HDFSBuilderWrapper::getKinitCmd() -{ - - if (hadoop_kerberos_keytab.empty() || hadoop_kerberos_principal.empty()) - { - throw Exception("Not enough parameters to run kinit", - ErrorCodes::NO_ELEMENTS_IN_CONFIG); - } - - WriteBufferFromOwnString ss; - - String cache_name = hadoop_security_kerberos_ticket_cache_path.empty() ? - String() : - (String(" -c \"") + hadoop_security_kerberos_ticket_cache_path + "\""); - - // command to run looks like - // kinit -R -t /keytab_dir/clickhouse.keytab -k somebody@TEST.CLICKHOUSE.TECH || .. - ss << hadoop_kerberos_kinit_command << cache_name << - " -R -t \"" << hadoop_kerberos_keytab << "\" -k " << hadoop_kerberos_principal << - "|| " << hadoop_kerberos_kinit_command << cache_name << " -t \"" << - hadoop_kerberos_keytab << "\" -k " << hadoop_kerberos_principal; - return ss.str(); -} - +#if USE_KRB5 void HDFSBuilderWrapper::runKinit() { - String cmd = getKinitCmd(); - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "running kinit: {}", cmd); - - std::unique_lock lck(kinit_mtx); - - auto command = ShellCommand::execute(cmd); - auto status = command->tryWait(); - if (status) + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); + try { - throw Exception("kinit failure: " + cmd, ErrorCodes::BAD_ARGUMENTS); + kerberosInit(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } + catch (const DB::Exception & e) + { + throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); + } + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); } +#endif // USE_KRB5 HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) { @@ -184,16 +171,16 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A } } + #if USE_KRB5 if (builder.need_kinit) { builder.runKinit(); } + #endif // USE_KRB5 return builder; } -std::mutex HDFSBuilderWrapper::kinit_mtx; - HDFSFSPtr createHDFSFS(hdfsBuilder * builder) { HDFSFSPtr fs(hdfsBuilderConnect(builder)); diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index 0523849abe5..9eb2dfd3e46 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -69,10 +68,6 @@ public: private: void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false); - String getKinitCmd(); - - void runKinit(); - // hdfs builder relies on an external config data storage std::pair& keep(const String & k, const String & v) { @@ -80,14 +75,15 @@ private: } hdfsBuilder * hdfs_builder; + std::vector> config_stor; + + #if USE_KRB5 + void runKinit(); String hadoop_kerberos_keytab; String hadoop_kerberos_principal; - String hadoop_kerberos_kinit_command = "kinit"; String hadoop_security_kerberos_ticket_cache_path; - - static std::mutex kinit_mtx; - std::vector> config_stor; bool need_kinit{false}; + #endif // USE_KRB5 }; using HDFSFSPtr = std::unique_ptr, detail::HDFSFsDeleter>; diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 6046dd58677..b717d373598 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -37,6 +37,7 @@ #include #include #include +#include namespace DB { @@ -766,8 +767,20 @@ Pipe StorageHive::read( sources_info->hive_metastore_client = hive_metastore_client; sources_info->partition_name_types = partition_name_types; - Block sample_block; const auto header_block = storage_snapshot->metadata->getSampleBlock(); + bool support_subset_columns = supportsSubcolumns(); + + auto settings = context_->getSettingsRef(); + auto case_insensitive_matching = [&]() -> bool + { + if (format_name == "Parquet") + return settings.input_format_parquet_case_insensitive_column_matching; + else if (format_name == "ORC") + return settings.input_format_orc_case_insensitive_column_matching; + return false; + }; + Block sample_block; + NestedColumnExtractHelper nested_columns_extractor(header_block, case_insensitive_matching()); for (const auto & column : column_names) { if (header_block.has(column)) @@ -775,7 +788,15 @@ Pipe StorageHive::read( sample_block.insert(header_block.getByName(column)); continue; } - + else if (support_subset_columns) + { + auto subset_column = nested_columns_extractor.extractColumn(column); + if (subset_column) + { + sample_block.insert(std::move(*subset_column)); + continue; + } + } if (column == "_path") sources_info->need_path_column = true; if (column == "_file") diff --git a/src/Storages/Hive/StorageHive.h b/src/Storages/Hive/StorageHive.h index d92d2dbd745..fd806553a86 100644 --- a/src/Storages/Hive/StorageHive.h +++ b/src/Storages/Hive/StorageHive.h @@ -42,6 +42,9 @@ public: String getName() const override { return "Hive"; } bool supportsIndexForIn() const override { return true; } + + bool supportsSubcolumns() const override { return true; } + bool mayBenefitFromIndexForIn( const ASTPtr & /* left_in_operand */, ContextPtr /* query_context */, diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index cd6c49d6e3b..fc29769790d 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -14,7 +14,8 @@ #include #include #include -#include +#include +#include namespace DB @@ -24,6 +25,7 @@ namespace ErrorCodes extern const int TABLE_IS_DROPPED; extern const int NOT_IMPLEMENTED; extern const int DEADLOCK_AVOIDED; + extern const int CANNOT_RESTORE_TABLE; } bool IStorage::isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const @@ -248,48 +250,21 @@ bool IStorage::isStaticStorage() const return false; } -ASTPtr IStorage::getCreateQueryForBackup(const ContextPtr & context, DatabasePtr * database) const +void IStorage::adjustCreateQueryForBackup(ASTPtr &) const { - auto table_id = getStorageID(); - auto db = DatabaseCatalog::instance().tryGetDatabase(table_id.getDatabaseName()); - if (!db) - throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", table_id.database_name, table_id.table_name); - ASTPtr query = db->tryGetCreateTableQuery(table_id.getTableName(), context); - if (!query) - throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", table_id.database_name, table_id.table_name); - - /// We don't want to see any UUIDs in backup (after RESTORE the table will have another UUID anyway). - auto & create = query->as(); - create.uuid = UUIDHelpers::Nil; - create.to_inner_uuid = UUIDHelpers::Nil; - - /// If this is a definition of a system table we'll remove columns and comment because they're excessive for backups. - if (create.storage && create.storage->engine && create.storage->engine->name.starts_with("System")) - { - create.reset(create.columns_list); - create.reset(create.comment); - } - - if (database) - *database = db; - - return query; -} - -ASTPtr IStorage::getCreateQueryForBackup(const BackupEntriesCollector & backup_entries_collector) const -{ - DatabasePtr database; - auto query = getCreateQueryForBackup(backup_entries_collector.getContext(), &database); - database->checkCreateTableQueryForBackup(query, backup_entries_collector); - return query; } void IStorage::backupData(BackupEntriesCollector &, const String &, const std::optional &) { } -void IStorage::restoreDataFromBackup(RestorerFromBackup &, const String &, const std::optional &) +void IStorage::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional &) { + /// If an inherited class doesn't override restoreDataFromBackup() that means it doesn't backup any data. + auto filenames = restorer.getBackup()->listFiles(data_path_in_backup); + if (!filenames.empty()) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: Folder {} in backup must be empty", + getStorageID().getFullTableName(), data_path_in_backup); } std::string PrewhereInfo::dump() const @@ -297,6 +272,11 @@ std::string PrewhereInfo::dump() const WriteBufferFromOwnString ss; ss << "PrewhereDagInfo\n"; + if (row_level_filter) + { + ss << "row_level_filter " << row_level_filter->dumpDAG() << "\n"; + } + if (prewhere_actions) { ss << "prewhere_actions " << prewhere_actions->dumpDAG() << "\n"; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index a655da4473b..e265c94eb11 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -223,10 +223,8 @@ public: /// Initially reserved virtual column name may be shadowed by real column. bool isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const; - /// Returns a slightly changed version of the CREATE TABLE query which must be written to a backup. - /// The function can throw `TABLE_IS_DROPPED` if this storage is not attached to a database. - virtual ASTPtr getCreateQueryForBackup(const ContextPtr & context, DatabasePtr * database) const; - virtual ASTPtr getCreateQueryForBackup(const BackupEntriesCollector & backup_entries_collector) const; + /// Modify a CREATE TABLE query to make a variant which must be written to a backup. + virtual void adjustCreateQueryForBackup(ASTPtr & create_query) const; /// Makes backup entries to backup the data of this storage. virtual void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions); @@ -234,6 +232,9 @@ public: /// Extracts data from the backup and put it to the storage. virtual void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions); + /// Returns true if the storage supports backup/restore for specific partitions. + virtual bool supportsBackupPartition() const { return false; } + private: StorageID storage_id; @@ -396,7 +397,7 @@ public: */ virtual void drop() {} - virtual void dropInnerTableIfAny(bool /* no_delay */, ContextPtr /* context */) {} + virtual void dropInnerTableIfAny(bool /* sync */, ContextPtr /* context */) {} /** Clear the table data and leave it empty. * Must be called under exclusive lock (lockExclusively). diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index d03db010a1f..058cc5ff7de 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -45,7 +45,9 @@ #include #include - +#if USE_KRB5 +#include +#endif // USE_KRB5 namespace CurrentMetrics { @@ -517,6 +519,33 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + #if USE_KRB5 + if (conf.has_property("sasl.kerberos.kinit.cmd")) + LOG_WARNING(log, "sasl.kerberos.kinit.cmd configuration parameter is ignored."); + + conf.set("sasl.kerberos.kinit.cmd",""); + conf.set("sasl.kerberos.min.time.before.relogin","0"); + + if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) + { + String keytab = conf.get("sasl.kerberos.keytab"); + String principal = conf.get("sasl.kerberos.principal"); + LOG_DEBUG(log, "Running KerberosInit"); + try + { + kerberosInit(keytab,principal); + } + catch (const Exception & e) + { + LOG_ERROR(log, "KerberosInit failure: {}", getExceptionMessage(e, false)); + } + LOG_DEBUG(log, "Finished KerberosInit"); + } + #else // USE_KRB5 + if (conf.has_property("sasl.kerberos.keytab") || conf.has_property("sasl.kerberos.principal")) + LOG_WARNING(log, "Kerberos-related parameters are ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 + // Update consumer topic-specific configuration for (const auto & topic : topics) { diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 75b2f981389..6a079aa832f 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -328,7 +328,7 @@ StorageLiveView::StorageLiveView( blocks_metadata_ptr = std::make_shared(); active_ptr = std::make_shared(true); - periodic_refresh_task = getContext()->getSchedulePool().createTask("LieViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); }); + periodic_refresh_task = getContext()->getSchedulePool().createTask("LiveViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); }); periodic_refresh_task->deactivate(); } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 083cbc90cb1..dc80b0aafb6 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -615,36 +615,50 @@ void DataPartStorageOnDisk::backup( TemporaryFilesOnDisks & temp_dirs, const MergeTreeDataPartChecksums & checksums, const NameSet & files_without_checksums, + const String & path_in_backup, BackupEntries & backup_entries) const { - auto disk = volume->getDisk(); + fs::path part_path_on_disk = fs::path{root_path} / part_dir; + fs::path part_path_in_backup = fs::path{path_in_backup} / part_dir; + auto disk = volume->getDisk(); auto temp_dir_it = temp_dirs.find(disk); if (temp_dir_it == temp_dirs.end()) - temp_dir_it = temp_dirs.emplace(disk, std::make_shared(disk, "tmp/backup_")).first; + temp_dir_it = temp_dirs.emplace(disk, std::make_shared(disk, "tmp/backup/")).first; auto temp_dir_owner = temp_dir_it->second; fs::path temp_dir = temp_dir_owner->getPath(); - - fs::path temp_part_dir = temp_dir / part_dir; + fs::path temp_part_dir = temp_dir / part_path_in_backup.relative_path(); disk->createDirectories(temp_part_dir); + /// For example, + /// part_path_in_backup = /data/test/table/0_1_1_0 + /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0 + /// tmp_part_dir = tmp/backup/1aaaaaa/data/test/table/0_1_1_0 + /// Or, for projections: + /// part_path_in_backup = /data/test/table/0_1_1_0/prjmax.proj + /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0/prjmax.proj + /// tmp_part_dir = tmp/backup/1aaaaaa/data/test/table/0_1_1_0/prjmax.proj + for (const auto & [filepath, checksum] : checksums.files) { - String relative_filepath = fs::path(part_dir) / filepath; - String full_filepath = fs::path(root_path) / part_dir / filepath; + if (filepath.ends_with(".proj")) + continue; /// Skip *.proj files - they're actually directories and will be handled. + String filepath_on_disk = part_path_on_disk / filepath; + String filepath_in_backup = part_path_in_backup / filepath; String hardlink_filepath = temp_part_dir / filepath; - disk->createHardLink(full_filepath, hardlink_filepath); + + disk->createHardLink(filepath_on_disk, hardlink_filepath); UInt128 file_hash{checksum.file_hash.first, checksum.file_hash.second}; backup_entries.emplace_back( - relative_filepath, + filepath_in_backup, std::make_unique(disk, hardlink_filepath, checksum.file_size, file_hash, temp_dir_owner)); } for (const auto & filepath : files_without_checksums) { - String relative_filepath = fs::path(part_dir) / filepath; - String full_filepath = fs::path(root_path) / part_dir / filepath; - backup_entries.emplace_back(relative_filepath, std::make_unique(disk, full_filepath)); + String filepath_on_disk = part_path_on_disk / filepath; + String filepath_in_backup = part_path_in_backup / filepath; + backup_entries.emplace_back(filepath_in_backup, std::make_unique(disk, filepath_on_disk)); } } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index d6fcb2f1442..bb1a8879d63 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -88,6 +88,7 @@ public: TemporaryFilesOnDisks & temp_dirs, const MergeTreeDataPartChecksums & checksums, const NameSet & files_without_checksums, + const String & path_in_backup, BackupEntries & backup_entries) const override; DataPartStoragePtr freeze( diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 0e165e74ed0..72810680812 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -173,6 +173,7 @@ public: TemporaryFilesOnDisks & temp_dirs, const MergeTreeDataPartChecksums & checksums, const NameSet & files_without_checksums, + const String & path_in_backup, BackupEntries & backup_entries) const = 0; /// Creates hardlinks into 'to/dir_path' for every file in data part. diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index 7c5977b5cb2..b13db9c3255 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -89,8 +89,6 @@ protected: using ColumnPosition = std::optional; ColumnPosition findColumnForOffsets(const String & column_name) const; - friend class MergeTreeRangeReader::DelayedStream; - private: /// Alter conversions, which must be applied on fly if required MergeTreeData::AlterConversions alter_conversions; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index f6baae723c9..40f23fe5294 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1300,7 +1300,7 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, ContextPtr context, } else { - DataTypePtr common_type = tryGetLeastSupertype({key_expr_type_not_null, const_type}); + DataTypePtr common_type = tryGetLeastSupertype(DataTypes{key_expr_type_not_null, const_type}); if (!common_type) return false; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 048c460c549..4121262e7b5 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -264,7 +264,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite /// Task is not needed merge_task.reset(); - storage.merger_mutator.renameMergedTemporaryPart(part, parts, NO_TRANSACTION_PTR, transaction_ptr.get()); + storage.merger_mutator.renameMergedTemporaryPart(part, parts, NO_TRANSACTION_PTR, *transaction_ptr); try { diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index c6a719fbc67..cc5e87956a1 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -115,7 +115,11 @@ void MergePlainMergeTreeTask::prepare() void MergePlainMergeTreeTask::finish() { new_part = merge_task->getFuture().get(); - storage.merger_mutator.renameMergedTemporaryPart(new_part, future_part->parts, txn, nullptr); + + MergeTreeData::Transaction transaction(storage, txn.get()); + storage.merger_mutator.renameMergedTemporaryPart(new_part, future_part->parts, txn, transaction); + transaction.commit(); + write_part_log({}); storage.incrementMergedPartsProfileEvent(new_part->getType()); } diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index ca5e7393666..31ca533a9ec 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -74,14 +74,27 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( prewhere_actions = std::make_unique(); if (prewhere_info->row_level_filter) - prewhere_actions->row_level_filter = std::make_shared(prewhere_info->row_level_filter, actions_settings); + { + PrewhereExprStep row_level_filter_step + { + .actions = std::make_shared(prewhere_info->row_level_filter, actions_settings), + .column_name = prewhere_info->row_level_column_name, + .remove_column = true, + .need_filter = true + }; - prewhere_actions->prewhere_actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings); + prewhere_actions->steps.emplace_back(std::move(row_level_filter_step)); + } - prewhere_actions->row_level_column_name = prewhere_info->row_level_column_name; - prewhere_actions->prewhere_column_name = prewhere_info->prewhere_column_name; - prewhere_actions->remove_prewhere_column = prewhere_info->remove_prewhere_column; - prewhere_actions->need_filter = prewhere_info->need_filter; + PrewhereExprStep prewhere_step + { + .actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings), + .column_name = prewhere_info->prewhere_column_name, + .remove_column = prewhere_info->remove_prewhere_column, + .need_filter = prewhere_info->need_filter + }; + + prewhere_actions->steps.emplace_back(std::move(prewhere_step)); } } @@ -204,30 +217,78 @@ Chunk MergeTreeBaseSelectProcessor::generate() void MergeTreeBaseSelectProcessor::initializeRangeReaders(MergeTreeReadTask & current_task) { + MergeTreeRangeReader* prev_reader = nullptr; + bool last_reader = false; + if (prewhere_info) { - if (reader->getColumns().empty()) - { - current_task.range_reader = MergeTreeRangeReader(pre_reader.get(), nullptr, prewhere_actions.get(), true, non_const_virtual_column_names); - } - else - { - MergeTreeRangeReader * pre_reader_ptr = nullptr; - if (pre_reader != nullptr) - { - current_task.pre_range_reader = MergeTreeRangeReader(pre_reader.get(), nullptr, prewhere_actions.get(), false, non_const_virtual_column_names); - pre_reader_ptr = ¤t_task.pre_range_reader; - } + if (prewhere_actions->steps.size() != pre_reader_for_step.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PREWHERE steps count mismatch, actions: {}, readers: {}", + prewhere_actions->steps.size(), pre_reader_for_step.size()); - current_task.range_reader = MergeTreeRangeReader(reader.get(), pre_reader_ptr, nullptr, true, non_const_virtual_column_names); + + for (size_t i = 0; i < prewhere_actions->steps.size(); ++i) + { + last_reader = reader->getColumns().empty() && (i + 1 == prewhere_actions->steps.size()); + current_task.pre_range_readers.push_back( + MergeTreeRangeReader(pre_reader_for_step[i].get(), prev_reader, &prewhere_actions->steps[i], last_reader, non_const_virtual_column_names)); + + prev_reader = ¤t_task.pre_range_readers.back(); } + + } + + if (!last_reader) + { + current_task.range_reader = MergeTreeRangeReader(reader.get(), prev_reader, nullptr, true, non_const_virtual_column_names); } else { - current_task.range_reader = MergeTreeRangeReader(reader.get(), nullptr, nullptr, true, non_const_virtual_column_names); + /// If all columns are read by pre_range_readers than move last pre_range_reader into range_reader + current_task.range_reader = std::move(current_task.pre_range_readers.back()); + current_task.pre_range_readers.pop_back(); } } +static UInt64 estimateNumRows(const MergeTreeReadTask & current_task, UInt64 current_preferred_block_size_bytes, + UInt64 current_max_block_size_rows, UInt64 current_preferred_max_column_in_block_size_bytes, double min_filtration_ratio) +{ + const MergeTreeRangeReader & current_reader = current_task.range_reader; + + if (!current_task.size_predictor) + return static_cast(current_max_block_size_rows); + + /// Calculates number of rows will be read using preferred_block_size_bytes. + /// Can't be less than avg_index_granularity. + size_t rows_to_read = current_task.size_predictor->estimateNumRows(current_preferred_block_size_bytes); + if (!rows_to_read) + return rows_to_read; + auto total_row_in_current_granule = current_reader.numRowsInCurrentGranule(); + rows_to_read = std::max(total_row_in_current_granule, rows_to_read); + + if (current_preferred_max_column_in_block_size_bytes) + { + /// Calculates number of rows will be read using preferred_max_column_in_block_size_bytes. + auto rows_to_read_for_max_size_column + = current_task.size_predictor->estimateNumRowsForMaxSizeColumn(current_preferred_max_column_in_block_size_bytes); + double filtration_ratio = std::max(min_filtration_ratio, 1.0 - current_task.size_predictor->filtered_rows_ratio); + auto rows_to_read_for_max_size_column_with_filtration + = static_cast(rows_to_read_for_max_size_column / filtration_ratio); + + /// If preferred_max_column_in_block_size_bytes is used, number of rows to read can be less than current_index_granularity. + rows_to_read = std::min(rows_to_read, rows_to_read_for_max_size_column_with_filtration); + } + + auto unread_rows_in_current_granule = current_reader.numPendingRowsInCurrentGranule(); + if (unread_rows_in_current_granule >= rows_to_read) + return rows_to_read; + + const MergeTreeIndexGranularity & index_granularity = current_task.data_part->index_granularity; + + return index_granularity.countMarksForRows(current_reader.currentMark(), rows_to_read, current_reader.numReadRowsInCurrentGranule()); +} + Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() { @@ -237,45 +298,10 @@ Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() const UInt64 current_max_block_size_rows = max_block_size_rows; const UInt64 current_preferred_block_size_bytes = preferred_block_size_bytes; const UInt64 current_preferred_max_column_in_block_size_bytes = preferred_max_column_in_block_size_bytes; - const MergeTreeIndexGranularity & index_granularity = task->data_part->index_granularity; const double min_filtration_ratio = 0.00001; - auto estimate_num_rows = [current_preferred_block_size_bytes, current_max_block_size_rows, - &index_granularity, current_preferred_max_column_in_block_size_bytes, min_filtration_ratio]( - MergeTreeReadTask & current_task, MergeTreeRangeReader & current_reader) - { - if (!current_task.size_predictor) - return static_cast(current_max_block_size_rows); - - /// Calculates number of rows will be read using preferred_block_size_bytes. - /// Can't be less than avg_index_granularity. - size_t rows_to_read = current_task.size_predictor->estimateNumRows(current_preferred_block_size_bytes); - if (!rows_to_read) - return rows_to_read; - auto total_row_in_current_granule = current_reader.numRowsInCurrentGranule(); - rows_to_read = std::max(total_row_in_current_granule, rows_to_read); - - if (current_preferred_max_column_in_block_size_bytes) - { - /// Calculates number of rows will be read using preferred_max_column_in_block_size_bytes. - auto rows_to_read_for_max_size_column - = current_task.size_predictor->estimateNumRowsForMaxSizeColumn(current_preferred_max_column_in_block_size_bytes); - double filtration_ratio = std::max(min_filtration_ratio, 1.0 - current_task.size_predictor->filtered_rows_ratio); - auto rows_to_read_for_max_size_column_with_filtration - = static_cast(rows_to_read_for_max_size_column / filtration_ratio); - - /// If preferred_max_column_in_block_size_bytes is used, number of rows to read can be less than current_index_granularity. - rows_to_read = std::min(rows_to_read, rows_to_read_for_max_size_column_with_filtration); - } - - auto unread_rows_in_current_granule = current_reader.numPendingRowsInCurrentGranule(); - if (unread_rows_in_current_granule >= rows_to_read) - return rows_to_read; - - return index_granularity.countMarksForRows(current_reader.currentMark(), rows_to_read, current_reader.numReadRowsInCurrentGranule()); - }; - - UInt64 recommended_rows = estimate_num_rows(*task, task->range_reader); + UInt64 recommended_rows = estimateNumRows(*task, current_preferred_block_size_bytes, + current_max_block_size_rows, current_preferred_max_column_in_block_size_bytes, min_filtration_ratio); UInt64 rows_to_read = std::max(static_cast(1), std::min(current_max_block_size_rows, recommended_rows)); auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); @@ -602,9 +628,12 @@ std::unique_ptr MergeTreeBaseSelectProcessor::getSi const Block & sample_block) { const auto & required_column_names = task_columns.columns.getNames(); - const auto & required_pre_column_names = task_columns.pre_columns.getNames(); NameSet complete_column_names(required_column_names.begin(), required_column_names.end()); - complete_column_names.insert(required_pre_column_names.begin(), required_pre_column_names.end()); + for (const auto & pre_columns_per_step : task_columns.pre_columns) + { + const auto & required_pre_column_names = pre_columns_per_step.getNames(); + complete_column_names.insert(required_pre_column_names.begin(), required_pre_column_names.end()); + } return std::make_unique( data_part, Names(complete_column_names.begin(), complete_column_names.end()), sample_block); diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index ac01221ff26..368dce7deaa 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -115,7 +115,7 @@ protected: using MergeTreeReaderPtr = std::unique_ptr; MergeTreeReaderPtr reader; - MergeTreeReaderPtr pre_reader; + std::vector pre_reader_for_step; MergeTreeReadTaskPtr task; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index f74823eaec2..50f4c34f004 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -5,6 +5,9 @@ #include #include #include +#include +#include + #include @@ -131,12 +134,12 @@ NameSet injectRequiredColumns( MergeTreeReadTask::MergeTreeReadTask( const MergeTreeData::DataPartPtr & data_part_, const MarkRanges & mark_ranges_, size_t part_index_in_query_, - const Names & ordered_names_, const NameSet & column_name_set_, const NamesAndTypesList & columns_, - const NamesAndTypesList & pre_columns_, bool remove_prewhere_column_, bool should_reorder_, + const Names & ordered_names_, const NameSet & column_name_set_, const MergeTreeReadTaskColumns & task_columns_, + bool remove_prewhere_column_, MergeTreeBlockSizePredictorPtr && size_predictor_) : data_part{data_part_}, mark_ranges{mark_ranges_}, part_index_in_query{part_index_in_query_}, - ordered_names{ordered_names_}, column_name_set{column_name_set_}, columns{columns_}, pre_columns{pre_columns_}, - remove_prewhere_column{remove_prewhere_column_}, should_reorder{should_reorder_}, size_predictor{std::move(size_predictor_)} + ordered_names{ordered_names_}, column_name_set{column_name_set_}, task_columns{task_columns_}, + remove_prewhere_column{remove_prewhere_column_}, size_predictor{std::move(size_predictor_)} { } @@ -276,34 +279,40 @@ MergeTreeReadTaskColumns getReadTaskColumns( Names pre_column_names; /// inject columns required for defaults evaluation - bool should_reorder = !injectRequiredColumns( - storage, storage_snapshot, data_part, with_subcolumns, column_names).empty(); + injectRequiredColumns( + storage, storage_snapshot, data_part, with_subcolumns, column_names); + + MergeTreeReadTaskColumns result; + auto options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects(); + if (with_subcolumns) + options.withSubcolumns(); if (prewhere_info) { - pre_column_names = prewhere_info->prewhere_actions->getRequiredColumnsNames(); + NameSet pre_name_set; + /// Add column reading steps: + /// 1. Columns for row level filter if (prewhere_info->row_level_filter) { - NameSet names(pre_column_names.begin(), pre_column_names.end()); - - for (auto & name : prewhere_info->row_level_filter->getRequiredColumnsNames()) - { - if (!names.contains(name)) - pre_column_names.push_back(name); - } + Names row_filter_column_names = prewhere_info->row_level_filter->getRequiredColumnsNames(); + result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, row_filter_column_names)); + pre_name_set.insert(row_filter_column_names.begin(), row_filter_column_names.end()); } - if (pre_column_names.empty()) - pre_column_names.push_back(column_names[0]); + /// 2. Columns for prewhere + Names all_pre_column_names = prewhere_info->prewhere_actions->getRequiredColumnsNames(); const auto injected_pre_columns = injectRequiredColumns( - storage, storage_snapshot, data_part, with_subcolumns, pre_column_names); + storage, storage_snapshot, data_part, with_subcolumns, all_pre_column_names); - if (!injected_pre_columns.empty()) - should_reorder = true; - - const NameSet pre_name_set(pre_column_names.begin(), pre_column_names.end()); + for (const auto & name : all_pre_column_names) + { + if (pre_name_set.contains(name)) + continue; + pre_column_names.push_back(name); + pre_name_set.insert(name); + } Names post_column_names; for (const auto & name : column_names) @@ -313,17 +322,23 @@ MergeTreeReadTaskColumns getReadTaskColumns( column_names = post_column_names; } - MergeTreeReadTaskColumns result; - NamesAndTypesList all_columns; + result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, pre_column_names)); - auto options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects(); - if (with_subcolumns) - options.withSubcolumns(); - - result.pre_columns = storage_snapshot->getColumnsByNames(options, pre_column_names); + /// 3. Rest of the requested columns result.columns = storage_snapshot->getColumnsByNames(options, column_names); - result.should_reorder = should_reorder; return result; } + +std::string MergeTreeReadTaskColumns::dump() const +{ + WriteBufferFromOwnString s; + for (size_t i = 0; i < pre_columns.size(); ++i) + { + s << "STEP " << i << ": " << pre_columns[i].toString() << "\n"; + } + s << "COLUMNS: " << columns.toString() << "\n"; + return s.str(); +} + } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 1c347970558..f9f82dbd1f2 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -30,6 +30,16 @@ NameSet injectRequiredColumns( Names & columns); +struct MergeTreeReadTaskColumns +{ + /// column names to read during WHERE + NamesAndTypesList columns; + /// column names to read during each PREWHERE step + std::vector pre_columns; + + std::string dump() const; +}; + /// A batch of work for MergeTreeThreadSelectProcessor struct MergeTreeReadTask { @@ -43,39 +53,27 @@ struct MergeTreeReadTask const Names & ordered_names; /// used to determine whether column should be filtered during PREWHERE or WHERE const NameSet & column_name_set; - /// column names to read during WHERE - const NamesAndTypesList & columns; - /// column names to read during PREWHERE - const NamesAndTypesList & pre_columns; + /// column names to read during PREWHERE and WHERE + const MergeTreeReadTaskColumns & task_columns; /// should PREWHERE column be returned to requesting side? const bool remove_prewhere_column; - /// resulting block may require reordering in accordance with `ordered_names` - const bool should_reorder; /// Used to satistfy preferred_block_size_bytes limitation MergeTreeBlockSizePredictorPtr size_predictor; /// Used to save current range processing status MergeTreeRangeReader range_reader; - MergeTreeRangeReader pre_range_reader; + /// Range readers for multiple filtering steps: row level security, PREWHERE etc. + /// NOTE: we take references to elements and push_back new elements, that's why it is a deque but noit a vector + std::deque pre_range_readers; bool isFinished() const { return mark_ranges.empty() && range_reader.isCurrentRangeFinished(); } MergeTreeReadTask( const MergeTreeData::DataPartPtr & data_part_, const MarkRanges & mark_ranges_, size_t part_index_in_query_, - const Names & ordered_names_, const NameSet & column_name_set_, const NamesAndTypesList & columns_, - const NamesAndTypesList & pre_columns_, bool remove_prewhere_column_, bool should_reorder_, + const Names & ordered_names_, const NameSet & column_name_set_, const MergeTreeReadTaskColumns & task_columns_, + bool remove_prewhere_column_, MergeTreeBlockSizePredictorPtr && size_predictor_); }; -struct MergeTreeReadTaskColumns -{ - /// column names to read during WHERE - NamesAndTypesList columns; - /// column names to read during PREWHERE - NamesAndTypesList pre_columns; - /// resulting block may require reordering in accordance with `ordered_names` - bool should_reorder = false; -}; - MergeTreeReadTaskColumns getReadTaskColumns( const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot, diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c24636a56f8..d4b2c8d177e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -96,7 +96,6 @@ namespace ProfileEvents extern const Event RejectedInserts; extern const Event DelayedInserts; extern const Event DelayedInsertsMilliseconds; - extern const Event DuplicatedInsertedBlocks; extern const Event InsertedWideParts; extern const Event InsertedCompactParts; extern const Event InsertedInMemoryParts; @@ -151,6 +150,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; extern const int INCORRECT_QUERY; + extern const int CANNOT_RESTORE_TABLE; } static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key, bool check_sample_column_is_correct) @@ -1325,20 +1325,29 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto deactivate_part = [&] (DataPartIteratorByStateAndInfo it) { + const DataPartPtr & part = *it; - (*it)->remove_time.store((*it)->modification_time, std::memory_order_relaxed); - auto creation_csn = (*it)->version.creation_csn.load(std::memory_order_relaxed); - if (creation_csn != Tx::RolledBackCSN && creation_csn != Tx::PrehistoricCSN && !(*it)->version.isRemovalTIDLocked()) + part->remove_time.store(part->modification_time, std::memory_order_relaxed); + auto creation_csn = part->version.creation_csn.load(std::memory_order_relaxed); + if (creation_csn != Tx::RolledBackCSN && creation_csn != Tx::PrehistoricCSN && !part->version.isRemovalTIDLocked()) { /// It's possible that covering part was created without transaction, /// but if covered part was created with transaction (i.e. creation_tid is not prehistoric), /// then it must have removal tid in metadata file. throw Exception(ErrorCodes::LOGICAL_ERROR, "Data part {} is Outdated and has creation TID {} and CSN {}, " "but does not have removal tid. It's a bug or a result of manual intervention.", - (*it)->name, (*it)->version.creation_tid, creation_csn); + part->name, part->version.creation_tid, creation_csn); } modifyPartState(it, DataPartState::Outdated); - removePartContributionToDataVolume(*it); + removePartContributionToDataVolume(part); + + /// Explicitly set removal_tid_lock for parts w/o transaction (i.e. w/o txn_version.txt) + /// to avoid keeping part forever (see VersionMetadata::canBeRemoved()) + if (!part->version.isRemovalTIDLocked()) + { + TransactionInfoContext transaction_context{getStorageID(), part->name}; + part->version.lockRemovalTID(Tx::PrehistoricTID, transaction_context); + } }; /// All parts are in "Active" state after loading @@ -2786,22 +2795,14 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( bool MergeTreeData::renameTempPartAndAdd( MutableDataPartPtr & part, - MergeTreeTransaction * txn, - SimpleIncrement * increment, - Transaction * out_transaction, - MergeTreeDeduplicationLog * deduplication_log, - std::string_view deduplication_token) + Transaction & out_transaction, + DataPartsLock & lock) { - if (out_transaction && &out_transaction->data != this) - throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", - ErrorCodes::LOGICAL_ERROR); - DataPartsVector covered_parts; - { - auto lock = lockParts(); - if (!renameTempPartAndReplace(part, txn, increment, out_transaction, lock, &covered_parts, deduplication_log, deduplication_token)) - return false; - } + + if (!renameTempPartAndReplaceImpl(part, out_transaction, lock, &covered_parts)) + return false; + if (!covered_parts.empty()) throw Exception("Added part " + part->name + " covers " + toString(covered_parts.size()) + " existing part(s) (including " + covered_parts[0]->name + ")", ErrorCodes::LOGICAL_ERROR); @@ -2809,29 +2810,10 @@ bool MergeTreeData::renameTempPartAndAdd( return true; } - -bool MergeTreeData::renameTempPartAndReplace( - MutableDataPartPtr & part, - MergeTreeTransaction * txn, - SimpleIncrement * increment, - Transaction * out_transaction, - std::unique_lock & lock, - DataPartsVector * out_covered_parts, - MergeTreeDeduplicationLog * deduplication_log, - std::string_view deduplication_token) +void MergeTreeData::checkPartCanBeAddedToTable(MutableDataPartPtr & part, DataPartsLock & lock) const { - if (out_transaction && &out_transaction->data != this) - throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", - ErrorCodes::LOGICAL_ERROR); - - if (txn) - transactions_enabled.store(true); - part->assertState({DataPartState::Temporary}); - MergeTreePartInfo part_info = part->info; - String part_name; - if (DataPartPtr existing_part_in_partition = getAnyPartInPartition(part->info.partition_id, lock)) { if (part->partition.value != existing_part_in_partition->partition.value) @@ -2841,21 +2823,7 @@ bool MergeTreeData::renameTempPartAndReplace( ErrorCodes::CORRUPTED_DATA); } - /** It is important that obtaining new block number and adding that block to parts set is done atomically. - * Otherwise there is race condition - merge of blocks could happen in interval that doesn't yet contain new part. - */ - if (increment) - { - part_info.min_block = part_info.max_block = increment->get(); - part_info.mutation = 0; /// it's equal to min_block by default - part_name = part->getNewName(part_info); - } - else /// Parts from ReplicatedMergeTree already have names - part_name = part->name; - - LOG_TRACE(log, "Renaming temporary part {} to {}.", part->data_part_storage->getPartDirectory(), part_name); - - if (auto it_duplicate = data_parts_by_info.find(part_info); it_duplicate != data_parts_by_info.end()) + if (auto it_duplicate = data_parts_by_info.find(part->info); it_duplicate != data_parts_by_info.end()) { String message = "Part " + (*it_duplicate)->getNameWithState() + " already exists"; @@ -2864,93 +2832,51 @@ bool MergeTreeData::renameTempPartAndReplace( throw Exception(message, ErrorCodes::DUPLICATE_DATA_PART); } +} + +void MergeTreeData::preparePartForCommit(MutableDataPartPtr & part, Transaction & out_transaction, bool need_rename) +{ + part->is_temp = false; + part->setState(DataPartState::PreActive); + + if (need_rename) + part->renameTo(part->name, true); + + data_parts_indexes.insert(part); + out_transaction.precommitted_parts.insert(part); +} + +bool MergeTreeData::renameTempPartAndReplaceImpl( + MutableDataPartPtr & part, + Transaction & out_transaction, + DataPartsLock & lock, + DataPartsVector * out_covered_parts) +{ + LOG_TRACE(log, "Renaming temporary part {} to {}.", part->data_part_storage->getPartDirectory(), part->name); + + if (&out_transaction.data != this) + throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", + ErrorCodes::LOGICAL_ERROR); + + checkPartCanBeAddedToTable(part, lock); DataPartPtr covering_part; - DataPartsVector covered_parts = getActivePartsToReplace(part_info, part_name, covering_part, lock); + DataPartsVector covered_parts = getActivePartsToReplace(part->info, part->name, covering_part, lock); if (covering_part) { - LOG_WARNING(log, "Tried to add obsolete part {} covered by {}", part_name, covering_part->getNameWithState()); + LOG_WARNING(log, "Tried to add obsolete part {} covered by {}", part->name, covering_part->getNameWithState()); return false; } - /// Deduplication log used only from non-replicated MergeTree. Replicated - /// tables have their own mechanism. We try to deduplicate at such deep - /// level, because only here we know real part name which is required for - /// deduplication. - if (deduplication_log) - { - const String block_id = part->getZeroLevelPartBlockID(deduplication_token); - auto res = deduplication_log->addPart(block_id, part_info); - if (!res.second) - { - ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); - LOG_INFO(log, "Block with ID {} already exists as part {}; ignoring it", block_id, res.first.getPartName()); - return false; - } - } - /// All checks are passed. Now we can rename the part on disk. /// So, we maintain invariant: if a non-temporary part in filesystem then it is in data_parts - /// - /// If out_transaction is null, we commit the part to the active set immediately, else add it to the transaction. - - part->name = part_name; - part->info = part_info; - part->is_temp = false; - part->setState(DataPartState::PreActive); - part->renameTo(part_name, true); - - auto part_it = data_parts_indexes.insert(part).first; - - if (out_transaction) - { - chassert(out_transaction->txn == txn); - out_transaction->precommitted_parts.insert(part); - } - else - { - /// FIXME Transactions: it's not the best place for checking and setting removal_tid, - /// because it's too optimistic. We should lock removal_tid of covered parts at the beginning of operation. - MergeTreeTransaction::addNewPartAndRemoveCovered(shared_from_this(), part, covered_parts, txn); - - size_t reduce_bytes = 0; - size_t reduce_rows = 0; - size_t reduce_parts = 0; - auto current_time = time(nullptr); - for (const DataPartPtr & covered_part : covered_parts) - { - covered_part->remove_time.store(current_time, std::memory_order_relaxed); - modifyPartState(covered_part, DataPartState::Outdated); - removePartContributionToColumnAndSecondaryIndexSizes(covered_part); - reduce_bytes += covered_part->getBytesOnDisk(); - reduce_rows += covered_part->rows_count; - ++reduce_parts; - } - - modifyPartState(part_it, DataPartState::Active); - addPartContributionToColumnAndSecondaryIndexSizes(part); - - if (covered_parts.empty()) - updateObjectColumns(*part_it, lock); - else - resetObjectColumnsFromActiveParts(lock); - - ssize_t diff_bytes = part->getBytesOnDisk() - reduce_bytes; - ssize_t diff_rows = part->rows_count - reduce_rows; - ssize_t diff_parts = 1 - reduce_parts; - increaseDataVolume(diff_bytes, diff_rows, diff_parts); - } - - auto part_in_memory = asInMemoryPart(part); - if (part_in_memory && getSettings()->in_memory_parts_enable_wal) - { - auto wal = getWriteAheadLog(); - wal->addPart(part_in_memory); - } + preparePartForCommit(part, out_transaction, /* need_rename = */ true); if (out_covered_parts) { + out_covered_parts->reserve(covered_parts.size()); + for (DataPartPtr & covered_part : covered_parts) out_covered_parts->emplace_back(std::move(covered_part)); } @@ -2958,24 +2884,26 @@ bool MergeTreeData::renameTempPartAndReplace( return true; } -MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( - MutableDataPartPtr & part, MergeTreeTransaction * txn, SimpleIncrement * increment, - Transaction * out_transaction, MergeTreeDeduplicationLog * deduplication_log) +MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplaceUnlocked( + MutableDataPartPtr & part, + Transaction & out_transaction, + DataPartsLock & lock) { - if (out_transaction && &out_transaction->data != this) - throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", - ErrorCodes::LOGICAL_ERROR); - DataPartsVector covered_parts; - { - auto lock = lockParts(); - renameTempPartAndReplace(part, txn, increment, out_transaction, lock, &covered_parts, deduplication_log); - } + renameTempPartAndReplaceImpl(part, out_transaction, lock, &covered_parts); + return covered_parts; } -void MergeTreeData::removePartsFromWorkingSet(MergeTreeTransaction * txn, const MergeTreeData::DataPartsVector & remove, bool clear_without_timeout, DataPartsLock & acquired_lock) +MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( + MutableDataPartPtr & part, + Transaction & out_transaction) +{ + auto part_lock = lockParts(); + return renameTempPartAndReplaceUnlocked(part, out_transaction, part_lock); +} +void MergeTreeData::removePartsFromWorkingSet(MergeTreeTransaction * txn, const MergeTreeData::DataPartsVector & remove, bool clear_without_timeout, DataPartsLock & acquired_lock) { if (txn) transactions_enabled.store(true); @@ -4066,14 +3994,23 @@ BackupEntries MergeTreeData::backupParts(const ContextPtr & local_context, const BackupEntries backup_entries; std::map> temp_dirs; - fs::path data_path_in_backup_fs = data_path_in_backup; for (const auto & part : data_parts) - part->data_part_storage->backup(temp_dirs, part->checksums, part->getFileNamesWithoutChecksums(), backup_entries); + { + part->data_part_storage->backup( + temp_dirs, part->checksums, part->getFileNamesWithoutChecksums(), data_path_in_backup, backup_entries); - /// TODO: try to write better code later. - for (auto & entry : backup_entries) - entry.first = data_path_in_backup_fs / entry.first; + auto projection_parts = part->getProjectionParts(); + for (const auto & [projection_name, projection_part] : projection_parts) + { + projection_part->data_part_storage->backup( + temp_dirs, + projection_part->checksums, + projection_part->getFileNamesWithoutChecksums(), + fs::path{data_path_in_backup} / part->name, + backup_entries); + } + } return backup_entries; } @@ -4156,7 +4093,10 @@ void MergeTreeData::restorePartsFromBackup(RestorerFromBackup & restorer, const { const auto part_info = MergeTreePartInfo::tryParsePartName(part_name, format_version); if (!part_info) - continue; + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File name {} doesn't look like the name of a part", + getStorageID().getFullTableName(), String{data_path_in_backup_fs / part_name}); + } if (partition_ids && !partition_ids->contains(part_info->partition_id)) continue; @@ -4189,27 +4129,39 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r auto disk = reservation->getDisk(); String part_name = part_info.getPartName(); - auto temp_part_dir_owner = std::make_shared(disk, relative_data_path + "restoring_" + part_name + "_"); - String temp_part_dir = temp_part_dir_owner->getPath(); + auto temp_part_dir_owner = std::make_shared(disk, fs::path{relative_data_path} / ("restoring_" + part_name + "_")); + fs::path temp_part_dir = temp_part_dir_owner->getPath(); disk->createDirectories(temp_part_dir); + std::unordered_set subdirs; - assert(temp_part_dir.starts_with(relative_data_path)); - String relative_temp_part_dir = temp_part_dir.substr(relative_data_path.size()); + /// temp_part_name = "restoring__", for example "restoring_0_1_1_0_1baaaaa" + String temp_part_name = temp_part_dir.filename(); for (const String & filename : filenames) { + /// Needs to create subdirectories before copying the files. Subdirectories are used to represent projections. + auto separator_pos = filename.rfind('/'); + if (separator_pos != String::npos) + { + String subdir = filename.substr(0, separator_pos); + if (subdirs.emplace(subdir).second) + disk->createDirectories(temp_part_dir / subdir); + } + + /// TODO Transactions: Decide what to do with version metadata (if any). Let's just skip it for now. + if (filename.ends_with(IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME)) + continue; + auto backup_entry = backup->readFile(part_path_in_backup_fs / filename); auto read_buffer = backup_entry->getReadBuffer(); - auto write_buffer = disk->writeFile(fs::path(temp_part_dir) / filename); + auto write_buffer = disk->writeFile(temp_part_dir / filename); copyData(*read_buffer, *write_buffer); reservation->update(reservation->getSize() - backup_entry->getSize()); } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, relative_temp_part_dir); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, temp_part_name); auto part = createPart(part_name, part_info, data_part_storage); - /// TODO Transactions: Decide what to do with version metadata (if any). Let's just remove it for now. - disk->removeFileIfExists(fs::path(temp_part_dir) / IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME); part->version.setCreationTID(Tx::PrehistoricTID, nullptr); part->loadColumnsChecksumsIndexes(false, true); @@ -4879,6 +4831,14 @@ MergeTreeData::DataPartPtr MergeTreeData::getAnyPartInPartition( } +MergeTreeData::Transaction::Transaction(MergeTreeData & data_, MergeTreeTransaction * txn_) + : data(data_) + , txn(txn_) +{ + if (txn) + data.transactions_enabled.store(true); +} + void MergeTreeData::Transaction::rollbackPartsToTemporaryState() { if (!isEmpty()) @@ -4922,9 +4882,12 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: if (!isEmpty()) { + auto settings = data.getSettings(); + MergeTreeData::WriteAheadLogPtr wal; auto parts_lock = acquired_parts_lock ? MergeTreeData::DataPartsLock() : data.lockParts(); auto * owing_parts_lock = acquired_parts_lock ? acquired_parts_lock : &parts_lock; + if (txn) { for (const DataPartPtr & part : precommitted_parts) @@ -4949,6 +4912,15 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: for (const DataPartPtr & part : precommitted_parts) { + auto part_in_memory = asInMemoryPart(part); + if (part_in_memory && settings->in_memory_parts_enable_wal) + { + if (!wal) + wal = data.getWriteAheadLog(); + + wal->addPart(part_in_memory); + } + DataPartPtr covering_part; DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock); if (covering_part) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 00a56de9142..9aa14367f80 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -252,7 +252,7 @@ public: class Transaction : private boost::noncopyable { public: - Transaction(MergeTreeData & data_, MergeTreeTransaction * txn_) : data(data_), txn(txn_) {} + Transaction(MergeTreeData & data_, MergeTreeTransaction * txn_); DataPartsVector commit(MergeTreeData::DataPartsLock * acquired_parts_lock = nullptr); @@ -549,37 +549,24 @@ public: /// Renames temporary part to a permanent part and adds it to the parts set. /// It is assumed that the part does not intersect with existing parts. - /// If increment != nullptr, part index is determining using increment. Otherwise part index remains unchanged. - /// If out_transaction != nullptr, adds the part in the PreActive state (the part will be added to the - /// active set later with out_transaction->commit()). - /// Else, commits the part immediately. + /// Adds the part in the PreActive state (the part will be added to the active set later with out_transaction->commit()). /// Returns true if part was added. Returns false if part is covered by bigger part. bool renameTempPartAndAdd( MutableDataPartPtr & part, - MergeTreeTransaction * txn, - SimpleIncrement * increment = nullptr, - Transaction * out_transaction = nullptr, - MergeTreeDeduplicationLog * deduplication_log = nullptr, - std::string_view deduplication_token = std::string_view()); + Transaction & transaction, + DataPartsLock & lock); /// The same as renameTempPartAndAdd but the block range of the part can contain existing parts. /// Returns all parts covered by the added part (in ascending order). - /// If out_transaction == nullptr, marks covered parts as Outdated. DataPartsVector renameTempPartAndReplace( - MutableDataPartPtr & part, MergeTreeTransaction * txn, SimpleIncrement * increment = nullptr, - Transaction * out_transaction = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr); - - /// Low-level version of previous one, doesn't lock mutex - /// FIXME Transactions: remove add_to_txn flag, maybe merge MergeTreeTransaction and Transaction - bool renameTempPartAndReplace( MutableDataPartPtr & part, - MergeTreeTransaction * txn, - SimpleIncrement * increment, - Transaction * out_transaction, - DataPartsLock & lock, - DataPartsVector * out_covered_parts = nullptr, - MergeTreeDeduplicationLog * deduplication_log = nullptr, - std::string_view deduplication_token = std::string_view()); + Transaction & out_transaction); + + /// Unlocked version of previous one. Useful when added multiple parts with a single lock. + DataPartsVector renameTempPartAndReplaceUnlocked( + MutableDataPartPtr & part, + Transaction & out_transaction, + DataPartsLock & lock); /// Remove parts from working set immediately (without wait for background /// process). Transfer part state to temporary. Have very limited usage only @@ -729,6 +716,9 @@ public: /// Extract data from the backup and put it to the storage. void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + /// Returns true if the storage supports backup/restore for specific partitions. + bool supportsBackupPartition() const override { return true; } + /// Moves partition to specified Disk void movePartitionToDisk(const ASTPtr & partition, const String & name, bool moving_part, ContextPtr context); @@ -1251,6 +1241,22 @@ protected: static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type); private: + + /// Checking that candidate part doesn't break invariants: correct partition and doesn't exist already + void checkPartCanBeAddedToTable(MutableDataPartPtr & part, DataPartsLock & lock) const; + + /// Preparing itself to be committed in memory: fill some fields inside part, add it to data_parts_indexes + /// in precommitted state and to transasction + void preparePartForCommit(MutableDataPartPtr & part, Transaction & out_transaction, bool need_rename); + + /// Low-level method for preparing parts for commit (in-memory). + /// FIXME Merge MergeTreeTransaction and Transaction + bool renameTempPartAndReplaceImpl( + MutableDataPartPtr & part, + Transaction & out_transaction, + DataPartsLock & lock, + DataPartsVector * out_covered_parts); + /// RAII Wrapper for atomic work with currently moving parts /// Acquire them in constructor and remove them in destructor /// Uses data.currently_moving_parts_mutex diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 77e3b574804..c740489b76d 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -541,7 +541,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart MergeTreeData::MutableDataPartPtr & new_data_part, const MergeTreeData::DataPartsVector & parts, const MergeTreeTransactionPtr & txn, - MergeTreeData::Transaction * out_transaction) + MergeTreeData::Transaction & out_transaction) { /// Some of source parts was possibly created in transaction, so non-transactional merge may break isolation. if (data.transactions_enabled.load(std::memory_order_relaxed) && !txn) @@ -549,7 +549,7 @@ MergeTreeData::DataPartPtr MergeTreeDataMergerMutator::renameMergedTemporaryPart "but transactions were enabled for this table"); /// Rename new part, add to the set and remove original parts. - auto replaced_parts = data.renameTempPartAndReplace(new_data_part, txn.get(), nullptr, out_transaction); + auto replaced_parts = data.renameTempPartAndReplace(new_data_part, out_transaction); /// Let's check that all original parts have been deleted and only them. if (replaced_parts.size() != parts.size()) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index e3d59a3522f..587b6b26347 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -133,7 +133,7 @@ public: MergeTreeData::MutableDataPartPtr & new_data_part, const MergeTreeData::DataPartsVector & parts, const MergeTreeTransactionPtr & txn, - MergeTreeData::Transaction * out_transaction = nullptr); + MergeTreeData::Transaction & out_transaction); /// The approximate amount of disk space needed for merge or mutation. With a surplus. diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 30fb3efcf0e..85231aca253 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -263,23 +263,22 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( Pipe ordinary_pipe; if (query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { - auto make_aggregator_params = [&](const Block & header_before_aggregation, bool projection) + auto make_aggregator_params = [&](bool projection) { - ColumnNumbers keys; - for (const auto & key : query_info.projection->aggregation_keys) - keys.push_back(header_before_aggregation.getPositionByName(key.name)); + const auto & keys = query_info.projection->aggregation_keys.getNames(); AggregateDescriptions aggregates = query_info.projection->aggregate_descriptions; - if (!projection) - { - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); - } + + /// This part is hacky. + /// We want AggregatingTransform to work with aggregate states instead of normal columns. + /// It is almost the same, just instead of adding new data to aggregation state we merge it with existing. + /// + /// It is needed because data in projection: + /// * is not merged completely (we may have states with the same key in different parts) + /// * is not split into buckets (so if we just use MergingAggregated, it will use single thread) + const bool only_merge = projection; Aggregator::Params params( - header_before_aggregation, keys, aggregates, query_info.projection->aggregate_overflow_row, @@ -293,23 +292,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( settings.max_threads, settings.min_free_disk_space_for_temporary_data, settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); - - bool only_merge = false; - if (projection) - { - /// The source header is also an intermediate header - params.intermediate_header = header_before_aggregation; - - /// This part is hacky. - /// We want AggregatingTransform to work with aggregate states instead of normal columns. - /// It is almost the same, just instead of adding new data to aggregation state we merge it with existing. - /// - /// It is needed because data in projection: - /// * is not merged completely (we may have states with the same key in different parts) - /// * is not split into buckets (so if we just use MergingAggregated, it will use single thread) - only_merge = true; - } + settings.min_count_to_compile_aggregate_expression, + only_merge); return std::make_pair(params, only_merge); }; @@ -343,10 +327,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( /// TODO apply optimize_aggregation_in_order here too (like below) auto build_aggregate_pipe = [&](Pipe & pipe, bool projection) { - auto [params, only_merge] = make_aggregator_params(pipe.getHeader(), projection); + auto [params, only_merge] = make_aggregator_params(projection); AggregatingTransformParamsPtr transform_params = std::make_shared( - std::move(params), aggregator_list_ptr, query_info.projection->aggregate_final, only_merge); + pipe.getHeader(), std::move(params), aggregator_list_ptr, query_info.projection->aggregate_final); pipe.resize(pipe.numOutputPorts(), true, true); @@ -371,7 +355,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( { auto add_aggregating_step = [&](QueryPlanPtr & query_plan, bool projection) { - auto [params, only_merge] = make_aggregator_params(query_plan->getCurrentDataStream().header, projection); + auto [params, only_merge] = make_aggregator_params(projection); auto merge_threads = num_streams; auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads @@ -385,19 +369,23 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( else group_by_info = nullptr; + // We don't have information regarding the `to_stage` of the query processing, only about `from_stage` (which is passed through `processed_stage` argument). + // Thus we cannot assign false here since it may be a query over distributed table. + const bool should_produce_results_in_order_of_bucket_number = true; + auto aggregating_step = std::make_unique( query_plan->getCurrentDataStream(), std::move(params), /* grouping_sets_params_= */ GroupingSetsParamsList{}, query_info.projection->aggregate_final, - only_merge, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, merge_threads, temporary_data_merge_threads, /* storage_has_evenly_distributed_read_= */ false, std::move(group_by_info), - std::move(group_by_sort_description)); + std::move(group_by_sort_description), + should_produce_results_in_order_of_bucket_number); query_plan->addStep(std::move(aggregating_step)); }; diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp index 961106af51b..280ce82cfce 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.cpp @@ -34,9 +34,9 @@ try : getSizePredictor(data_part, task_columns, sample_block); task = std::make_unique( - data_part, mark_ranges_for_task, part_index_in_query, ordered_names, column_name_set, task_columns.columns, - task_columns.pre_columns, prewhere_info && prewhere_info->remove_prewhere_column, - task_columns.should_reorder, std::move(size_predictor)); + data_part, mark_ranges_for_task, part_index_in_query, ordered_names, column_name_set, task_columns, + prewhere_info && prewhere_info->remove_prewhere_column, + std::move(size_predictor)); return true; } diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index d71dddc938c..6e14e9c7aa9 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -64,7 +66,7 @@ static void filterColumns(Columns & columns, const ColumnPtr & filter) } -static size_t getLastMark(const MergeTreeRangeReader::ReadResult::RangesInfo & ranges) +size_t MergeTreeRangeReader::ReadResult::getLastMark(const MergeTreeRangeReader::ReadResult::RangesInfo & ranges) { size_t current_task_last_mark = 0; for (const auto & mark_range : ranges) @@ -594,6 +596,7 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con return count; } +/// Filter size must match total_rows_per_granule void MergeTreeRangeReader::ReadResult::setFilter(const ColumnPtr & new_filter) { if (!new_filter && filter) @@ -644,7 +647,7 @@ size_t MergeTreeRangeReader::ReadResult::countBytesInResultFilter(const IColumn: MergeTreeRangeReader::MergeTreeRangeReader( IMergeTreeReader * merge_tree_reader_, MergeTreeRangeReader * prev_reader_, - const PrewhereExprInfo * prewhere_info_, + const PrewhereExprStep * prewhere_info_, bool last_reader_in_chain_, const Names & non_const_virtual_column_names_) : merge_tree_reader(merge_tree_reader_) @@ -672,17 +675,12 @@ MergeTreeRangeReader::MergeTreeRangeReader( if (prewhere_info) { - if (prewhere_info->row_level_filter) - { - prewhere_info->row_level_filter->execute(sample_block, true); - sample_block.erase(prewhere_info->row_level_column_name); - } + const auto & step = *prewhere_info; + if (step.actions) + step.actions->execute(sample_block, true); - if (prewhere_info->prewhere_actions) - prewhere_info->prewhere_actions->execute(sample_block, true); - - if (prewhere_info->remove_prewhere_column) - sample_block.erase(prewhere_info->prewhere_column_name); + if (step.remove_column) + sample_block.erase(step.column_name); } } @@ -983,11 +981,15 @@ void MergeTreeRangeReader::fillPartOffsetColumn(ReadResult & result, UInt64 lead result.columns.emplace_back(std::move(column)); } -Columns MergeTreeRangeReader::continueReadingChain(ReadResult & result, size_t & num_rows) +Columns MergeTreeRangeReader::continueReadingChain(const ReadResult & result, size_t & num_rows) { Columns columns; num_rows = 0; + /// No columns need to be read at this step? (only more filtering) + if (merge_tree_reader->getColumns().empty()) + return columns; + if (result.rowsPerGranule().empty()) { /// If zero rows were read on prev step, than there is no more rows to read. @@ -1001,7 +1003,7 @@ Columns MergeTreeRangeReader::continueReadingChain(ReadResult & result, size_t & const auto & rows_per_granule = result.rowsPerGranule(); const auto & started_ranges = result.startedRanges(); - size_t current_task_last_mark = getLastMark(started_ranges); + size_t current_task_last_mark = ReadResult::getLastMark(started_ranges); size_t next_range_to_start = 0; auto size = rows_per_granule.size(); @@ -1039,6 +1041,8 @@ static void checkCombinedFiltersSize(size_t bytes_in_first_filter, size_t second "does not match second filter size ({})", bytes_in_first_filter, second_filter_size); } +/// Second filter size must be equal to number of 1s in the first filter. +/// The result size is equal to first filter size. static ColumnPtr combineFilters(ColumnPtr first, ColumnPtr second) { ConstantFilterDescription first_const_descr(*first); @@ -1099,13 +1103,17 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r const auto & header = merge_tree_reader->getColumns(); size_t num_columns = header.size(); - if (result.columns.size() != (num_columns + non_const_virtual_column_names.size())) - throw Exception("Invalid number of columns passed to MergeTreeRangeReader. " - "Expected " + toString(num_columns) + ", " - "got " + toString(result.columns.size()), ErrorCodes::LOGICAL_ERROR); + /// Check that we have columns from previous steps and newly read required columns + if (result.columns.size() < num_columns + non_const_virtual_column_names.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Invalid number of columns passed to MergeTreeRangeReader. Expected {}, got {}", + num_columns, result.columns.size()); - ColumnPtr filter; - ColumnPtr row_level_filter; + /// This filter has the size of total_rows_per granule. It is applied after reading contiguous chunks from + /// the start of each granule. + ColumnPtr combined_filter; + /// Filter computed at the current step. Its size is equal to num_rows which is <= total_rows_per_granule + ColumnPtr current_step_filter; size_t prewhere_column_pos; { @@ -1122,13 +1130,23 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r } } - for (auto name_and_type = header.begin(); pos < num_columns; ++pos, ++name_and_type) + for (auto name_and_type = header.begin(); name_and_type != header.end() && pos < result.columns.size(); ++pos, ++name_and_type) block.insert({result.columns[pos], name_and_type->type, name_and_type->name}); for (const auto & column_name : non_const_virtual_column_names) { + if (block.has(column_name)) + continue; + if (column_name == "_part_offset") + { + if (pos >= result.columns.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Invalid number of columns passed to MergeTreeRangeReader. Expected {}, got {}", + num_columns, result.columns.size()); + block.insert({result.columns[pos], std::make_shared(), column_name}); + } else throw Exception("Unexpected non-const virtual column: " + column_name, ErrorCodes::LOGICAL_ERROR); ++pos; @@ -1137,58 +1155,37 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r /// Columns might be projected out. We need to store them here so that default columns can be evaluated later. result.block_before_prewhere = block; - if (prewhere_info->row_level_filter) - { - prewhere_info->row_level_filter->execute(block); - auto row_level_filter_pos = block.getPositionByName(prewhere_info->row_level_column_name); - row_level_filter = block.getByPosition(row_level_filter_pos).column; - block.erase(row_level_filter_pos); + if (prewhere_info->actions) + prewhere_info->actions->execute(block); - auto columns = block.getColumns(); - filterColumns(columns, row_level_filter); - if (columns.empty()) - block = block.cloneEmpty(); - else - block.setColumns(columns); - } - - prewhere_info->prewhere_actions->execute(block); - - prewhere_column_pos = block.getPositionByName(prewhere_info->prewhere_column_name); + prewhere_column_pos = block.getPositionByName(prewhere_info->column_name); result.columns.clear(); result.columns.reserve(block.columns()); for (auto & col : block) result.columns.emplace_back(std::move(col.column)); - filter.swap(result.columns[prewhere_column_pos]); + current_step_filter.swap(result.columns[prewhere_column_pos]); + combined_filter = current_step_filter; } if (result.getFilter()) { - /// TODO: implement for prewhere chain. - /// In order to do it we need combine filter and result.filter, where filter filters only '1' in result.filter. - throw Exception("MergeTreeRangeReader chain with several prewhere actions in not implemented.", - ErrorCodes::LOGICAL_ERROR); + ColumnPtr prev_filter = result.getFilterHolder(); + combined_filter = combineFilters(prev_filter, std::move(combined_filter)); } - if (filter && row_level_filter) - { - row_level_filter = combineFilters(std::move(row_level_filter), filter); - result.setFilter(row_level_filter); - } - else - result.setFilter(filter); + result.setFilter(combined_filter); /// If there is a WHERE, we filter in there, and only optimize IO and shrink columns here if (!last_reader_in_chain) - result.optimize(merge_tree_reader->canReadIncompleteGranules(), prewhere_info->row_level_filter == nullptr); + result.optimize(merge_tree_reader->canReadIncompleteGranules(), true); /// If we read nothing or filter gets optimized to nothing if (result.totalRowsPerGranule() == 0) result.setFilterConstFalse(); /// If we need to filter in PREWHERE - else if (prewhere_info->need_filter || result.need_filter || prewhere_info->row_level_filter) + else if (prewhere_info->need_filter || result.need_filter) { /// If there is a filter and without optimized if (result.getFilter() && last_reader_in_chain) @@ -1208,10 +1205,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r /// filter might be shrunk while columns not const auto * result_filter = result.getFilterOriginal(); - if (row_level_filter) - filterColumns(result.columns, filter); - else - filterColumns(result.columns, result_filter->getData()); + filterColumns(result.columns, current_step_filter); result.need_filter = true; @@ -1234,22 +1228,22 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r /// Check if the PREWHERE column is needed if (!result.columns.empty()) { - if (prewhere_info->remove_prewhere_column) + if (prewhere_info->remove_column) result.columns.erase(result.columns.begin() + prewhere_column_pos); else result.columns[prewhere_column_pos] = - getSampleBlock().getByName(prewhere_info->prewhere_column_name).type-> + getSampleBlock().getByName(prewhere_info->column_name).type-> createColumnConst(result.num_rows, 1u)->convertToFullColumnIfConst(); } } /// Filter in WHERE instead else { - if (prewhere_info->remove_prewhere_column) + if (prewhere_info->remove_column) result.columns.erase(result.columns.begin() + prewhere_column_pos); else { - auto type = getSampleBlock().getByName(prewhere_info->prewhere_column_name).type; + auto type = getSampleBlock().getByName(prewhere_info->column_name).type; ColumnWithTypeAndName col(result.getFilterHolder()->convertToFullColumnIfConst(), std::make_shared(), ""); result.columns[prewhere_column_pos] = castColumn(col, type); result.clearFilter(); // Acting as a flag to not filter in PREWHERE @@ -1257,4 +1251,20 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r } } +std::string PrewhereExprInfo::dump() const +{ + WriteBufferFromOwnString s; + + for (size_t i = 0; i < steps.size(); ++i) + { + s << "STEP " << i << ":\n" + << " ACTIONS: " << (steps[i].actions ? steps[i].actions->dumpActions() : "nullptr") << "\n" + << " COLUMN: " << steps[i].column_name << "\n" + << " REMOVE_COLUMN: " << steps[i].remove_column << "\n" + << " NEED_FILTER: " << steps[i].need_filter << "\n"; + } + + return s.str(); +} + } diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.h b/src/Storages/MergeTree/MergeTreeRangeReader.h index 21ed35e6a78..b4fb8913122 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -18,18 +18,20 @@ using PrewhereInfoPtr = std::shared_ptr; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; +struct PrewhereExprStep +{ + ExpressionActionsPtr actions; + String column_name; + bool remove_column = false; + bool need_filter = false; +}; + /// The same as PrewhereInfo, but with ExpressionActions instead of ActionsDAG struct PrewhereExprInfo { - /// Actions for row level security filter. Applied separately before prewhere_actions. - /// This actions are separate because prewhere condition should not be executed over filtered rows. - ExpressionActionsPtr row_level_filter; - /// Actions which are executed on block in order to get filter column for prewhere step. - ExpressionActionsPtr prewhere_actions; - String row_level_column_name; - String prewhere_column_name; - bool remove_prewhere_column = false; - bool need_filter = false; + std::vector steps; + + std::string dump() const; }; /// MergeTreeReader iterator which allows sequential reading for arbitrary number of rows between pairs of marks in the same part. @@ -41,7 +43,7 @@ public: MergeTreeRangeReader( IMergeTreeReader * merge_tree_reader_, MergeTreeRangeReader * prev_reader_, - const PrewhereExprInfo * prewhere_info_, + const PrewhereExprStep * prewhere_info_, bool last_reader_in_chain_, const Names & non_const_virtual_column_names); @@ -57,6 +59,7 @@ public: bool isCurrentRangeFinished() const; bool isInitialized() const { return is_initialized; } +private: /// Accumulates sequential read() requests to perform a large read instead of multiple small reads class DelayedStream { @@ -144,10 +147,23 @@ public: size_t ceilRowsToCompleteGranules(size_t rows_num) const; }; +public: /// Statistics after next reading step. class ReadResult { public: + Columns columns; + size_t num_rows = 0; + + /// The number of rows were added to block as a result of reading chain. + size_t numReadRows() const { return num_read_rows; } + /// The number of bytes read from disk. + size_t numBytesRead() const { return num_bytes_read; } + + private: + /// Only MergeTreeRangeReader is supposed to access ReadResult internals. + friend class MergeTreeRangeReader; + using NumRows = std::vector; struct RangeInfo @@ -161,13 +177,11 @@ public: const RangesInfo & startedRanges() const { return started_ranges; } const NumRows & rowsPerGranule() const { return rows_per_granule; } + static size_t getLastMark(const MergeTreeRangeReader::ReadResult::RangesInfo & ranges); + /// The number of rows were read at LAST iteration in chain. <= num_added_rows + num_filtered_rows. size_t totalRowsPerGranule() const { return total_rows_per_granule; } - /// The number of rows were added to block as a result of reading chain. - size_t numReadRows() const { return num_read_rows; } size_t numRowsToSkipInLastGranule() const { return num_rows_to_skip_in_last_granule; } - /// The number of bytes read from disk. - size_t numBytesRead() const { return num_bytes_read; } /// Filter you need to apply to newly-read columns in order to add them to block. const ColumnUInt8 * getFilterOriginal() const { return filter_original ? filter_original : filter; } const ColumnUInt8 * getFilter() const { return filter; } @@ -195,13 +209,12 @@ public: size_t countBytesInResultFilter(const IColumn::Filter & filter); - Columns columns; - size_t num_rows = 0; + /// If this flag is false than filtering form PREWHERE can be delayed and done in WHERE + /// to reduce memory copies and applying heavy filters multiple times bool need_filter = false; Block block_before_prewhere; - private: RangesInfo started_ranges; /// The number of rows read from each granule. /// Granule here is not number of rows between two marks @@ -234,16 +247,15 @@ public: const Block & getSampleBlock() const { return sample_block; } private: - ReadResult startReadingChain(size_t max_rows, MarkRanges & ranges); - Columns continueReadingChain(ReadResult & result, size_t & num_rows); + Columns continueReadingChain(const ReadResult & result, size_t & num_rows); void executePrewhereActionsAndFilterColumns(ReadResult & result); void fillPartOffsetColumn(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset); IMergeTreeReader * merge_tree_reader = nullptr; const MergeTreeIndexGranularity * index_granularity = nullptr; MergeTreeRangeReader * prev_reader = nullptr; /// If not nullptr, read from prev_reader firstly. - const PrewhereExprInfo * prewhere_info; + const PrewhereExprStep * prewhere_info; Stream stream; diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index d9a1f742609..d44d250149e 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -135,13 +135,15 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(size_t min_marks_to_read, size_t } } - auto curr_task_size_predictor = !per_part_size_predictor[part_idx] ? nullptr - : std::make_unique(*per_part_size_predictor[part_idx]); /// make a copy + const auto & per_part = per_part_params[part_idx]; + + auto curr_task_size_predictor = !per_part.size_predictor ? nullptr + : std::make_unique(*per_part.size_predictor); /// make a copy return std::make_unique( part.data_part, ranges_to_get_from_part, part.part_index_in_query, ordered_names, - per_part_column_name_set[part_idx], per_part_columns[part_idx], per_part_pre_columns[part_idx], - prewhere_info && prewhere_info->remove_prewhere_column, per_part_should_reorder[part_idx], std::move(curr_task_size_predictor)); + per_part.column_name_set, per_part.task_columns, + prewhere_info && prewhere_info->remove_prewhere_column, std::move(curr_task_size_predictor)); } Block MergeTreeReadPool::getHeader() const @@ -216,15 +218,14 @@ std::vector MergeTreeReadPool::fillPerPartInfo(const RangesInDataParts & auto size_predictor = !predict_block_size_bytes ? nullptr : MergeTreeBaseSelectProcessor::getSizePredictor(part.data_part, task_columns, sample_block); - per_part_size_predictor.emplace_back(std::move(size_predictor)); + auto & per_part = per_part_params.emplace_back(); + + per_part.size_predictor = std::move(size_predictor); /// will be used to distinguish between PREWHERE and WHERE columns when applying filter const auto & required_column_names = task_columns.columns.getNames(); - per_part_column_name_set.emplace_back(required_column_names.begin(), required_column_names.end()); - - per_part_pre_columns.push_back(std::move(task_columns.pre_columns)); - per_part_columns.push_back(std::move(task_columns.columns)); - per_part_should_reorder.push_back(task_columns.should_reorder); + per_part.column_name_set = {required_column_names.begin(), required_column_names.end()}; + per_part.task_columns = std::move(task_columns); parts_with_idx.push_back({ part.data_part, part.part_index_in_query }); } diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index b9db1254e08..d882c0d761f 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -99,11 +99,16 @@ private: const Names column_names; bool do_not_steal_tasks; bool predict_block_size_bytes; - std::vector per_part_column_name_set; - std::vector per_part_columns; - std::vector per_part_pre_columns; - std::vector per_part_should_reorder; - std::vector per_part_size_predictor; + + struct PerPartParams + { + MergeTreeReadTaskColumns task_columns; + NameSet column_name_set; + MergeTreeBlockSizePredictorPtr size_predictor; + }; + + std::vector per_part_params; + PrewhereInfoPtr prewhere_info; struct Part diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 93c1e23884a..07fe187332e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -67,9 +67,12 @@ size_t MergeTreeReaderWide::readRows( size_t read_rows = 0; try { - size_t num_columns = columns.size(); + size_t num_columns = res_columns.size(); checkNumberOfColumns(num_columns); + if (num_columns == 0) + return max_rows_to_read; + std::unordered_map caches; std::unordered_set prefetched_streams; diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 1a2ab8bff5b..84548d357b7 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -31,8 +31,8 @@ try task = std::make_unique( data_part, mark_ranges_for_task, part_index_in_query, ordered_names, column_name_set, - task_columns.columns, task_columns.pre_columns, prewhere_info && prewhere_info->remove_prewhere_column, - task_columns.should_reorder, std::move(size_predictor)); + task_columns, prewhere_info && prewhere_info->remove_prewhere_column, + std::move(size_predictor)); return true; } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 1d41c2c254d..aa6c457f412 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -66,10 +66,16 @@ void MergeTreeSelectProcessor::initializeReaders() reader = data_part->getReader(task_columns.columns, storage_snapshot->getMetadataForQuery(), all_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, {}, {}); - if (prewhere_info) - pre_reader = data_part->getReader(task_columns.pre_columns, storage_snapshot->getMetadataForQuery(), - all_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, {}, {}); + pre_reader_for_step.clear(); + if (prewhere_info) + { + for (const auto & pre_columns_for_step : task_columns.pre_columns) + { + pre_reader_for_step.push_back(data_part->getReader(pre_columns_for_step, storage_snapshot->getMetadataForQuery(), + all_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, {}, {})); + } + } } @@ -80,7 +86,7 @@ void MergeTreeSelectProcessor::finish() * buffers don't waste memory. */ reader.reset(); - pre_reader.reset(); + pre_reader_for_step.clear(); data_part.reset(); } diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index fbc916ddb2c..2f860b34fd5 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -3,6 +3,10 @@ #include #include +namespace ProfileEvents +{ + extern const Event DuplicatedInsertedBlocks; +} namespace DB { @@ -133,8 +137,41 @@ void MergeTreeSink::finishDelayedChunk() auto & part = partition.temp_part.part; + bool added = false; + + { + auto lock = storage.lockParts(); + storage.fillNewPartName(part, lock); + + auto * deduplication_log = storage.getDeduplicationLog(); + if (deduplication_log) + { + const String block_id = part->getZeroLevelPartBlockID(partition.block_dedup_token); + auto res = deduplication_log->addPart(block_id, part->info); + if (!res.second) + { + ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); + LOG_INFO(storage.log, "Block with ID {} already exists as part {}; ignoring it", block_id, res.first.getPartName()); + } + else + { + MergeTreeData::Transaction transaction(storage, context->getCurrentTransaction().get()); + added = storage.renameTempPartAndAdd(part, transaction, lock); + transaction.commit(&lock); + + } + } + else + { + MergeTreeData::Transaction transaction(storage, context->getCurrentTransaction().get()); + added = storage.renameTempPartAndAdd(part, transaction, lock); + transaction.commit(&lock); + } + + } + /// Part can be deduplicated, so increment counters and add to part log only if it's really added - if (storage.renameTempPartAndAdd(part, context->getCurrentTransaction().get(), &storage.increment, nullptr, storage.getDeduplicationLog(), partition.block_dedup_token)) + if (added) { PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns); storage.incrementInsertedPartsProfileEvent(part->getType()); diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp index 063f018b1a4..574ce4479f2 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp @@ -111,14 +111,20 @@ void MergeTreeThreadSelectProcessor::finalizeNewTask() owned_uncompressed_cache = storage.getContext()->getUncompressedCache(); owned_mark_cache = storage.getContext()->getMarkCache(); - reader = task->data_part->getReader(task->columns, metadata_snapshot, task->mark_ranges, + reader = task->data_part->getReader(task->task_columns.columns, metadata_snapshot, task->mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, IMergeTreeReader::ValueSizeMap{}, profile_callback); + pre_reader_for_step.clear(); if (prewhere_info) - pre_reader = task->data_part->getReader(task->pre_columns, metadata_snapshot, task->mark_ranges, - owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, - IMergeTreeReader::ValueSizeMap{}, profile_callback); + { + for (const auto & pre_columns_per_step : task->task_columns.pre_columns) + { + pre_reader_for_step.push_back(task->data_part->getReader(pre_columns_per_step, metadata_snapshot, task->mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, + IMergeTreeReader::ValueSizeMap{}, profile_callback)); + } + } } else { @@ -126,14 +132,20 @@ void MergeTreeThreadSelectProcessor::finalizeNewTask() if (part_name != last_readed_part_name) { /// retain avg_value_size_hints - reader = task->data_part->getReader(task->columns, metadata_snapshot, task->mark_ranges, + reader = task->data_part->getReader(task->task_columns.columns, metadata_snapshot, task->mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, reader->getAvgValueSizeHints(), profile_callback); + pre_reader_for_step.clear(); if (prewhere_info) - pre_reader = task->data_part->getReader(task->pre_columns, metadata_snapshot, task->mark_ranges, - owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, - reader->getAvgValueSizeHints(), profile_callback); + { + for (const auto & pre_columns_per_step : task->task_columns.pre_columns) + { + pre_reader_for_step.push_back(task->data_part->getReader(pre_columns_per_step, metadata_snapshot, task->mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, + reader->getAvgValueSizeHints(), profile_callback)); + } + } } } @@ -144,7 +156,7 @@ void MergeTreeThreadSelectProcessor::finalizeNewTask() void MergeTreeThreadSelectProcessor::finish() { reader.reset(); - pre_reader.reset(); + pre_reader_for_step.clear(); } diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 403d77165d4..ac4de51bbd9 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -171,7 +171,7 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit { new_part = mutate_task->getFuture().get(); - storage.renameTempPartAndReplace(new_part, NO_TRANSACTION_RAW, nullptr, transaction_ptr.get()); + storage.renameTempPartAndReplace(new_part, *transaction_ptr); try { diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 80a33bfe0e3..6214919337b 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -83,8 +83,12 @@ bool MutatePlainMergeTreeTask::executeStep() new_part = mutate_task->getFuture().get(); + + MergeTreeData::Transaction transaction(storage, merge_mutate_entry->txn.get()); /// FIXME Transactions: it's too optimistic, better to lock parts before starting transaction - storage.renameTempPartAndReplace(new_part, merge_mutate_entry->txn.get()); + storage.renameTempPartAndReplace(new_part, transaction); + transaction.commit(); + storage.updateMutationEntriesErrors(future_part, true, ""); write_part_log({}); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index add1ba875aa..f6c80baba05 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1103,7 +1103,9 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry continue; /// Parts are not disjoint, so new_part_name either contains or covers future_part. - chassert(future_part.contains(result_part) || result_part.contains(future_part)); + if (!(future_part.contains(result_part) || result_part.contains(future_part))) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected non-disjoint parts: {} and {}", future_part_elem.first, new_part_name); + /// We cannot execute `entry` (or upgrade its actual_part_name to `new_part_name`) /// while any covered or covering parts are processed. /// But we also cannot simply return true and postpone entry processing, because it may lead to kind of livelock. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 4f500208215..dfb5eb0bd69 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -189,7 +189,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() } catch (...) { - std::unique_lock lock(storage.last_queue_update_exception_lock); + std::lock_guard lock(storage.last_queue_update_exception_lock); storage.last_queue_update_exception = getCurrentExceptionMessage(false); throw; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index d217e16c830..07e21def184 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -476,7 +476,8 @@ void ReplicatedMergeTreeSink::commitPart( try { - renamed = storage.renameTempPartAndAdd(part, NO_TRANSACTION_RAW, nullptr, &transaction); + auto lock = storage.lockParts(); + renamed = storage.renameTempPartAndAdd(part, transaction, lock); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 7dee7b8d0f8..ea90179caa3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -353,4 +354,123 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl return diff; } +StorageInMemoryMetadata ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata(const ColumnsDescription & new_columns, ContextPtr context, const StorageInMemoryMetadata & old_metadata) const +{ + StorageInMemoryMetadata new_metadata = old_metadata; + new_metadata.columns = new_columns; + + if (!empty()) + { + auto parse_key_expr = [] (const String & key_expr) + { + ParserNotEmptyExpressionList parser(false); + auto new_sorting_key_expr_list = parseQuery(parser, key_expr, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + + ASTPtr order_by_ast; + if (new_sorting_key_expr_list->children.size() == 1) + order_by_ast = new_sorting_key_expr_list->children[0]; + else + { + auto tuple = makeASTFunction("tuple"); + tuple->arguments->children = new_sorting_key_expr_list->children; + order_by_ast = tuple; + } + return order_by_ast; + }; + + if (sorting_key_changed) + { + auto order_by_ast = parse_key_expr(new_sorting_key); + + new_metadata.sorting_key.recalculateWithNewAST(order_by_ast, new_metadata.columns, context); + + if (new_metadata.primary_key.definition_ast == nullptr) + { + /// Primary and sorting key become independent after this ALTER so we have to + /// save the old ORDER BY expression as the new primary key. + auto old_sorting_key_ast = old_metadata.getSortingKey().definition_ast; + new_metadata.primary_key = KeyDescription::getKeyFromAST( + old_sorting_key_ast, new_metadata.columns, context); + } + } + + if (sampling_expression_changed) + { + if (!new_sampling_expression.empty()) + { + auto sample_by_ast = parse_key_expr(new_sampling_expression); + new_metadata.sampling_key.recalculateWithNewAST(sample_by_ast, new_metadata.columns, context); + } + else /// SAMPLE BY was removed + { + new_metadata.sampling_key = {}; + } + } + + if (skip_indices_changed) + new_metadata.secondary_indices = IndicesDescription::parse(new_skip_indices, new_columns, context); + + if (constraints_changed) + new_metadata.constraints = ConstraintsDescription::parse(new_constraints); + + if (projections_changed) + new_metadata.projections = ProjectionsDescription::parse(new_projections, new_columns, context); + + if (ttl_table_changed) + { + if (!new_ttl_table.empty()) + { + ParserTTLExpressionList parser; + auto ttl_for_table_ast = parseQuery(parser, new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + ttl_for_table_ast, new_metadata.columns, context, new_metadata.primary_key); + } + else /// TTL was removed + { + new_metadata.table_ttl = TTLTableDescription{}; + } + } + } + + /// Changes in columns may affect following metadata fields + new_metadata.column_ttls_by_name.clear(); + for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) + { + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, context, new_metadata.primary_key); + new_metadata.column_ttls_by_name[name] = new_ttl_entry; + } + + if (new_metadata.partition_key.definition_ast != nullptr) + new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, context); + + if (!sorting_key_changed) /// otherwise already updated + new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, context); + + /// Primary key is special, it exists even if not defined + if (new_metadata.primary_key.definition_ast != nullptr) + { + new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, context); + } + else + { + new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, context); + new_metadata.primary_key.definition_ast = nullptr; + } + + if (!sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) + new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, context); + + if (!skip_indices_changed) /// otherwise already updated + { + for (auto & index : new_metadata.secondary_indices) + index.recalculateWithNewColumns(new_metadata.columns, context); + } + + if (!ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + new_metadata.table_ttl.definition_ast, new_metadata.columns, context, new_metadata.primary_key); + + return new_metadata; +} + } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index 6d510d20304..eb2d087e988 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -66,6 +66,8 @@ struct ReplicatedMergeTreeTableMetadata return !sorting_key_changed && !sampling_expression_changed && !skip_indices_changed && !projections_changed && !ttl_table_changed && !constraints_changed; } + + StorageInMemoryMetadata getNewMetadata(const ColumnsDescription & new_columns, ContextPtr context, const StorageInMemoryMetadata & old_metadata) const; }; void checkEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const; diff --git a/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp new file mode 100644 index 00000000000..45d667047af --- /dev/null +++ b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +std::optional tryExtractZkPathFromCreateQuery(const IAST & create_query, const ContextPtr & global_context) +{ + const auto * create = create_query.as(); + if (!create || !create->storage || !create->storage->engine) + return {}; + + /// Check if the table engine is one of the ReplicatedMergeTree family. + const auto & ast_engine = *create->storage->engine; + if (!ast_engine.name.starts_with("Replicated") || !ast_engine.name.ends_with("MergeTree")) + return {}; + + /// Get the first argument. + const auto * ast_arguments = typeid_cast(ast_engine.arguments.get()); + if (!ast_arguments || ast_arguments->children.empty()) + return {}; + + auto * ast_zk_path = typeid_cast(ast_arguments->children[0].get()); + if (!ast_zk_path || (ast_zk_path->value.getType() != Field::Types::String)) + return {}; + + String zk_path = ast_zk_path->value.safeGet(); + + /// Expand macros. + Macros::MacroExpansionInfo info; + info.table_id.table_name = create->getTable(); + info.table_id.database_name = create->getDatabase(); + info.table_id.uuid = create->uuid; + auto database = DatabaseCatalog::instance().tryGetDatabase(info.table_id.database_name); + if (database && database->getEngineName() == "Replicated") + { + info.shard = getReplicatedDatabaseShardName(database); + info.replica = getReplicatedDatabaseReplicaName(database); + } + + try + { + zk_path = global_context->getMacros()->expand(zk_path, info); + } + catch (...) + { + return {}; /// Couldn't expand macros. + } + + return zk_path; +} + +} diff --git a/src/Storages/MergeTree/extractZkPathFromCreateQuery.h b/src/Storages/MergeTree/extractZkPathFromCreateQuery.h new file mode 100644 index 00000000000..e22f76d2cd5 --- /dev/null +++ b/src/Storages/MergeTree/extractZkPathFromCreateQuery.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +class IAST; +class Context; +using ContextPtr = std::shared_ptr; + +/// Extracts a zookeeper path from a specified CREATE TABLE query. Returns std::nullopt if fails. +/// The function takes the first argument of the ReplicatedMergeTree table engine and expands macros in it. +/// It works like a part of what the create() function in registerStorageMergeTree.cpp does but in a simpler manner. +std::optional tryExtractZkPathFromCreateQuery(const IAST & create_query, const ContextPtr & global_context); + +} diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 73b9b364f7f..e52a0fed674 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -304,12 +304,22 @@ static StoragePtr create(const StorageFactory::Arguments & args) arg_idx, e.message(), getMergeTreeVerboseHelp(is_extended_storage_def)); } } + else if (!args.attach && !args.getLocalContext()->getSettingsRef().allow_deprecated_syntax_for_merge_tree) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "This syntax for *MergeTree engine is deprecated. " + "Use extended storage definition syntax with ORDER BY/PRIMARY KEY clause." + "See also allow_deprecated_syntax_for_merge_tree setting."); + } /// For Replicated. String zookeeper_path; String replica_name; StorageReplicatedMergeTree::RenamingRestrictions renaming_restrictions = StorageReplicatedMergeTree::RenamingRestrictions::ALLOW_ANY; + bool is_on_cluster = args.getLocalContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; + bool is_replicated_database = args.getLocalContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && + DatabaseCatalog::instance().getDatabase(args.table_id.database_name)->getEngineName() == "Replicated"; + if (replicated) { bool has_arguments = arg_num + 2 <= arg_cnt; @@ -372,17 +382,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) throw Exception("Expected two string literal arguments: zookeeper_path and replica_name", ErrorCodes::BAD_ARGUMENTS); /// Allow implicit {uuid} macros only for zookeeper_path in ON CLUSTER queries - bool is_on_cluster = args.getLocalContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; - bool is_replicated_database = args.getLocalContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && - DatabaseCatalog::instance().getDatabase(args.table_id.database_name)->getEngineName() == "Replicated"; bool allow_uuid_macro = is_on_cluster || is_replicated_database || args.query.attach; /// Unfold {database} and {table} macro on table creation, so table can be renamed. if (!args.attach) { - if (is_replicated_database && !is_extended_storage_def) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Old syntax is not allowed for ReplicatedMergeTree tables in Replicated databases"); - Macros::MacroExpansionInfo info; /// NOTE: it's not recursive info.expand_special_macros_only = true; diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index 61a40c374eb..a57328fb402 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -271,8 +271,24 @@ void MaterializedPostgreSQLConsumer::readTupleData( } }; + std::exception_ptr error; for (int column_idx = 0; column_idx < num_columns; ++column_idx) - proccess_column_value(readInt8(message, pos, size), column_idx); + { + try + { + proccess_column_value(readInt8(message, pos, size), column_idx); + } + catch (...) + { + insertDefaultValue(buffer, column_idx); + /// Let's collect only the first exception. + /// This delaying of error throw is needed because + /// some errors can be ignored and just logged, + /// but in this case we need to finish insertion to all columns. + if (!error) + error = std::current_exception(); + } + } switch (type) { @@ -303,6 +319,9 @@ void MaterializedPostgreSQLConsumer::readTupleData( break; } } + + if (error) + std::rethrow_exception(error); } diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index 190ffabe2c1..cc80d567d1d 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -240,7 +240,7 @@ void StorageMaterializedPostgreSQL::shutdown() } -void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool no_delay, ContextPtr local_context) +void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool sync, ContextPtr local_context) { /// If it is a table with database engine MaterializedPostgreSQL - return, because delition of /// internal tables is managed there. @@ -252,7 +252,7 @@ void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool no_delay, ContextPt auto nested_table = tryGetNested() != nullptr; if (nested_table) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, getNestedStorageID(), no_delay); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, getNestedStorageID(), sync); } diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h index bb3836a8853..f1eea33d4b0 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h @@ -84,7 +84,7 @@ public: void shutdown() override; /// Used only for single MaterializedPostgreSQL storage. - void dropInnerTableIfAny(bool no_delay, ContextPtr local_context) override; + void dropInnerTableIfAny(bool sync, ContextPtr local_context) override; NamesAndTypesList getVirtuals() const override; diff --git a/src/Storages/RabbitMQ/RabbitMQSource.cpp b/src/Storages/RabbitMQ/RabbitMQSource.cpp index 71d80f0a632..f6e5bb84037 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSource.cpp @@ -142,7 +142,7 @@ Chunk RabbitMQSource::generateImpl() while (true) { - if (buffer->eof()) + if (buffer->queueEmpty()) break; auto new_rows = executor.execute(); diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index e2de5179990..3543085f5a0 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -47,6 +47,12 @@ ReadBufferFromRabbitMQConsumer::~ReadBufferFromRabbitMQConsumer() } +void ReadBufferFromRabbitMQConsumer::closeChannel() +{ + if (consumer_channel) + consumer_channel->close(); +} + void ReadBufferFromRabbitMQConsumer::subscribe() { for (const auto & queue_name : queues) diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 8a527011a3c..bd55d169744 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -3,18 +3,24 @@ #include #include #include -#include -#include #include namespace Poco { - class Logger; +class Logger; +} + +namespace AMQP +{ +class TcpChannel; } namespace DB { +class RabbitMQHandler; +using ChannelPtr = std::unique_ptr; + class ReadBufferFromRabbitMQConsumer : public ReadBuffer { @@ -52,11 +58,7 @@ public: ChannelPtr & getChannel() { return consumer_channel; } void setupChannel(); bool needChannelUpdate(); - void closeChannel() - { - if (consumer_channel) - consumer_channel->close(); - } + void closeChannel(); void updateQueues(std::vector & queues_) { queues = queues_; } size_t queuesCount() { return queues.size(); } diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 81cb58e4f5e..1bfaabe142b 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -336,7 +336,7 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , ContextPtr, TableExclusiveLockHolder &) { - std::unique_lock lock(rocksdb_ptr_mx); + std::lock_guard lock(rocksdb_ptr_mx); rocksdb_ptr->Close(); rocksdb_ptr = nullptr; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 1324ebf5b28..ac6ead54016 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -51,6 +51,7 @@ namespace ErrorCodes extern const int SIZES_OF_MARKS_FILES_ARE_INCONSISTENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; + extern const int CANNOT_RESTORE_TABLE; } /// NOTE: The lock `StorageLog::rwlock` is NOT kept locked while reading, @@ -921,11 +922,8 @@ std::optional StorageLog::totalBytes(const Settings &) const return total_bytes; } -void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) +void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - auto lock_timeout = getLockTimeout(backup_entries_collector.getContext()); loadMarks(lock_timeout); @@ -986,16 +984,16 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c } } -void StorageLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) +void StorageLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); + auto backup = restorer.getBackup(); + if (!backup->hasFiles(data_path_in_backup)) + return; if (!num_data_files) return; - auto backup = restorer.getBackup(); - if (!restorer.isNonEmptyTableAllowed() && total_bytes && backup->hasFiles(data_path_in_backup)) + if (!restorer.isNonEmptyTableAllowed() && total_bytes) RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); auto lock_timeout = getLockTimeout(restorer.getContext()); @@ -1024,6 +1022,11 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p for (const auto & data_file : data_files) { String file_path_in_backup = data_path_in_backup_fs / fileName(data_file.path); + if (!backup->fileExists(file_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), file_path_in_backup); + } auto backup_entry = backup->readFile(file_path_in_backup); auto in = backup_entry->getReadBuffer(); auto out = disk->writeFile(data_file.path, max_compress_block_size, WriteMode::Append); @@ -1035,6 +1038,11 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p /// Append marks. size_t num_extra_marks = 0; String file_path_in_backup = data_path_in_backup_fs / fileName(marks_file_path); + if (!backup->fileExists(file_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), file_path_in_backup); + } size_t file_size = backup->getFileSize(file_path_in_backup); if (file_size % (num_data_files * sizeof(Mark)) != 0) throw Exception("Size of marks file is inconsistent", ErrorCodes::SIZES_OF_MARKS_FILES_ARE_INCONSISTENT); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index d0685c263f8..b01415f9590 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -215,10 +215,10 @@ void StorageMaterializedView::drop() dropInnerTableIfAny(true, getContext()); } -void StorageMaterializedView::dropInnerTableIfAny(bool no_delay, ContextPtr local_context) +void StorageMaterializedView::dropInnerTableIfAny(bool sync, ContextPtr local_context) { if (has_inner_table && tryGetTargetTable()) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, no_delay); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync); } void StorageMaterializedView::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) @@ -421,6 +421,13 @@ void StorageMaterializedView::restoreDataFromBackup(RestorerFromBackup & restore return getTargetTable()->restoreDataFromBackup(restorer, data_path_in_backup, partitions); } +bool StorageMaterializedView::supportsBackupPartition() const +{ + if (hasInnerTable()) + return getTargetTable()->supportsBackupPartition(); + return false; +} + std::optional StorageMaterializedView::totalRows(const Settings & settings) const { if (hasInnerTable()) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 8aec0313ecb..1d8808b302e 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -42,7 +42,7 @@ public: SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; void drop() override; - void dropInnerTableIfAny(bool no_delay, ContextPtr local_context) override; + void dropInnerTableIfAny(bool sync, ContextPtr local_context) override; void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override; @@ -97,6 +97,7 @@ public: void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + bool supportsBackupPartition() const override; std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 5de8c3bda43..7baecaa594f 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -38,6 +38,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int CANNOT_RESTORE_TABLE; } @@ -479,24 +480,21 @@ namespace }; } -void StorageMemory::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) +void StorageMemory::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; backup_entries_collector.addBackupEntries( std::make_shared(getInMemoryMetadataPtr(), data.get(), data_path_in_backup, max_compress_block_size) ->getBackupEntries()); } -void StorageMemory::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) +void StorageMemory::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto backup = restorer.getBackup(); - if (!restorer.isNonEmptyTableAllowed() && total_size_bytes && backup->hasFiles(data_path_in_backup)) + if (!backup->hasFiles(data_path_in_backup)) + return; + + if (!restorer.isNonEmptyTableAllowed() && total_size_bytes) RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); restorer.addDataRestoreTask( @@ -514,6 +512,11 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat IndexForNativeFormat index; { String index_file_path = data_path_in_backup_fs / "index.mrk"; + if (!backup->fileExists(index_file_path)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), index_file_path); + } auto backup_entry = backup->readFile(index_file_path); auto in = backup_entry->getReadBuffer(); CompressedReadBuffer compressed_in{*in}; @@ -526,6 +529,11 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat size_t new_rows = 0; { String data_file_path = data_path_in_backup_fs / "data.bin"; + if (!backup->fileExists(data_file_path)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), data_file_path); + } auto backup_entry = backup->readFile(data_file_path); std::unique_ptr in = backup_entry->getReadBuffer(); std::optional temp_data_copy; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 4cebd95bab9..5fe7214194a 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1540,7 +1540,14 @@ PartitionCommandsResultInfo StorageMergeTree::attachPartition( loaded_parts[i]->storeVersionMetadata(); String old_name = renamed_parts.old_and_new_names[i].old_name; - renameTempPartAndAdd(loaded_parts[i], local_context->getCurrentTransaction().get(), &increment); + { + auto lock = lockParts(); + MergeTreeData::Transaction transaction(*this, local_context->getCurrentTransaction().get()); + fillNewPartName(loaded_parts[i], lock); + renameTempPartAndAdd(loaded_parts[i], transaction, lock); + transaction.commit(&lock); + } + renamed_parts.old_and_new_names[i].old_name.clear(); results.push_back(PartitionCommandResultInfo{ @@ -1612,10 +1619,15 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con auto data_parts_lock = lockParts(); + /** It is important that obtaining new block number and adding that block to parts set is done atomically. + * Otherwise there is race condition - merge of blocks could happen in interval that doesn't yet contain new part. + */ + for (auto part : dst_parts) + { + fillNewPartName(part, data_parts_lock); + renameTempPartAndReplaceUnlocked(part, transaction, data_parts_lock); + } /// Populate transaction - for (MutableDataPartPtr & part : dst_parts) - renameTempPartAndReplace(part, local_context->getCurrentTransaction().get(), &increment, &transaction, data_parts_lock); - transaction.commit(&data_parts_lock); /// If it is REPLACE (not ATTACH), remove all parts which max_block_number less then min_block_number of the first new block @@ -1688,14 +1700,15 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const auto src_data_parts_lock = lockParts(); auto dest_data_parts_lock = dest_table_storage->lockParts(); - std::mutex mutex; - DataPartsLock lock(mutex); + for (auto & part : dst_parts) + { + dest_table_storage->fillNewPartName(part, dest_data_parts_lock); + dest_table_storage->renameTempPartAndReplaceUnlocked(part, transaction, dest_data_parts_lock); + } - for (MutableDataPartPtr & part : dst_parts) - dest_table_storage->renameTempPartAndReplace(part, local_context->getCurrentTransaction().get(), &dest_table_storage->increment, &transaction, lock); - removePartsFromWorkingSet(local_context->getCurrentTransaction().get(), src_parts, true, lock); - transaction.commit(&lock); + removePartsFromWorkingSet(local_context->getCurrentTransaction().get(), src_parts, true, src_data_parts_lock); + transaction.commit(&src_data_parts_lock); } clearOldPartsFromFilesystem(); @@ -1785,7 +1798,13 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) { for (auto part : parts) - renameTempPartAndAdd(part, NO_TRANSACTION_RAW, &increment); + { + auto lock = lockParts(); + MergeTreeData::Transaction transaction(*this, NO_TRANSACTION_RAW); + fillNewPartName(part, lock); + renameTempPartAndAdd(part, transaction, lock); + transaction.commit(&lock); + } } @@ -1810,4 +1829,11 @@ std::unique_ptr StorageMergeTree::getDefaultSettings() const return std::make_unique(getContext()->getMergeTreeSettings()); } +void StorageMergeTree::fillNewPartName(MutableDataPartPtr & part, DataPartsLock &) +{ + part->info.min_block = part->info.max_block = increment.get(); + part->info.mutation = 0; + part->name = part->getNewName(part->info); +} + } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index ccfe03d012a..582962551d0 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -251,6 +251,8 @@ private: /// return any ids. std::optional getIncompleteMutationsStatus(Int64 mutation_version, std::set * mutation_ids = nullptr) const; + void fillNewPartName(MutableDataPartPtr & part, DataPartsLock & lock); + void startBackgroundMovesIfNeeded() override; /// Attaches restored parts to the storage. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e93399918ef..2ee1e233688 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -598,8 +599,19 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() auto zookeeper = getZooKeeper(); std::vector futures; - futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); + /// These 4 nodes used to be created in createNewZookeeperNodes() and they were moved to createTable() + /// This means that if the first replica creating the table metadata has an older version of CH (22.3 or previous) + /// there will be a time between its calls to `createTable` and `createNewZookeeperNodes` where the nodes won't exists + /// and that will cause issues in newer replicas + /// See https://github.com/ClickHouse/ClickHouse/issues/38600 for example + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum", String(), zkutil::CreateMode::Persistent)); + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/last_part", String(), zkutil::CreateMode::Persistent)); + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/failed_parts", String(), zkutil::CreateMode::Persistent)); + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/mutations", String(), zkutil::CreateMode::Persistent)); + + + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); /// Nodes for remote fs zero-copy replication const auto settings = getSettings(); if (settings->allow_remote_fs_zero_copy_replication) @@ -1089,123 +1101,8 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr void StorageReplicatedMergeTree::setTableStructure( ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff) { - StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - - new_metadata.columns = new_columns; - - if (!metadata_diff.empty()) - { - auto parse_key_expr = [] (const String & key_expr) - { - ParserNotEmptyExpressionList parser(false); - auto new_sorting_key_expr_list = parseQuery(parser, key_expr, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - - ASTPtr order_by_ast; - if (new_sorting_key_expr_list->children.size() == 1) - order_by_ast = new_sorting_key_expr_list->children[0]; - else - { - auto tuple = makeASTFunction("tuple"); - tuple->arguments->children = new_sorting_key_expr_list->children; - order_by_ast = tuple; - } - return order_by_ast; - }; - - if (metadata_diff.sorting_key_changed) - { - auto order_by_ast = parse_key_expr(metadata_diff.new_sorting_key); - auto & sorting_key = new_metadata.sorting_key; - auto & primary_key = new_metadata.primary_key; - - sorting_key.recalculateWithNewAST(order_by_ast, new_metadata.columns, getContext()); - - if (primary_key.definition_ast == nullptr) - { - /// Primary and sorting key become independent after this ALTER so we have to - /// save the old ORDER BY expression as the new primary key. - auto old_sorting_key_ast = old_metadata.getSortingKey().definition_ast; - primary_key = KeyDescription::getKeyFromAST( - old_sorting_key_ast, new_metadata.columns, getContext()); - } - } - - if (metadata_diff.sampling_expression_changed) - { - if (!metadata_diff.new_sampling_expression.empty()) - { - auto sample_by_ast = parse_key_expr(metadata_diff.new_sampling_expression); - new_metadata.sampling_key.recalculateWithNewAST(sample_by_ast, new_metadata.columns, getContext()); - } - else /// SAMPLE BY was removed - { - new_metadata.sampling_key = {}; - } - } - - if (metadata_diff.skip_indices_changed) - new_metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, getContext()); - - if (metadata_diff.constraints_changed) - new_metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); - - if (metadata_diff.projections_changed) - new_metadata.projections = ProjectionsDescription::parse(metadata_diff.new_projections, new_columns, getContext()); - - if (metadata_diff.ttl_table_changed) - { - if (!metadata_diff.new_ttl_table.empty()) - { - ParserTTLExpressionList parser; - auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - ttl_for_table_ast, new_metadata.columns, getContext(), new_metadata.primary_key); - } - else /// TTL was removed - { - new_metadata.table_ttl = TTLTableDescription{}; - } - } - } - - /// Changes in columns may affect following metadata fields - new_metadata.column_ttls_by_name.clear(); - for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) - { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, getContext(), new_metadata.primary_key); - new_metadata.column_ttls_by_name[name] = new_ttl_entry; - } - - if (new_metadata.partition_key.definition_ast != nullptr) - new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - - if (!metadata_diff.sorting_key_changed) /// otherwise already updated - new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - - /// Primary key is special, it exists even if not defined - if (new_metadata.primary_key.definition_ast != nullptr) - { - new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - } - else - { - new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, getContext()); - new_metadata.primary_key.definition_ast = nullptr; - } - - if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) - new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - - if (!metadata_diff.skip_indices_changed) /// otherwise already updated - { - for (auto & index : new_metadata.secondary_indices) - index.recalculateWithNewColumns(new_metadata.columns, getContext()); - } - - if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - new_metadata.table_ttl.definition_ast, new_metadata.columns, getContext(), new_metadata.primary_key); + StorageInMemoryMetadata new_metadata = metadata_diff.getNewMetadata(new_columns, getContext(), old_metadata); /// Even if the primary/sorting/partition keys didn't change we must reinitialize it /// because primary/partition key column types might have changed. @@ -1657,7 +1554,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) Transaction transaction(*this, NO_TRANSACTION_RAW); part->version.setCreationTID(Tx::PrehistoricTID, nullptr); - renameTempPartAndReplace(part, NO_TRANSACTION_RAW, nullptr, &transaction); + renameTempPartAndReplace(part, transaction); checkPartChecksumsAndCommit(transaction, part); writePartLog(PartLogElement::Type::NEW_PART, {}, 0 /** log entry is fake so we don't measure the time */, @@ -2342,7 +2239,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) Coordination::Requests ops; for (PartDescriptionPtr & part_desc : final_parts) { - renameTempPartAndReplace(part_desc->res_part, NO_TRANSACTION_RAW, nullptr, &transaction); + renameTempPartAndReplace(part_desc->res_part, transaction); getCommitPartOps(ops, part_desc->res_part); lockSharedData(*part_desc->res_part, false, part_desc->hardlinked_files); @@ -2969,7 +2866,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke String StorageReplicatedMergeTree::getLastQueueUpdateException() const { - std::unique_lock lock(last_queue_update_exception_lock); + std::lock_guard lock(last_queue_update_exception_lock); return last_queue_update_exception; } @@ -2991,7 +2888,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - std::unique_lock lock(last_queue_update_exception_lock); + std::lock_guard lock(last_queue_update_exception_lock); last_queue_update_exception = getCurrentExceptionMessage(false); if (e.code == Coordination::Error::ZSESSIONEXPIRED) @@ -3006,7 +2903,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - std::unique_lock lock(last_queue_update_exception_lock); + std::lock_guard lock(last_queue_update_exception_lock); last_queue_update_exception = getCurrentExceptionMessage(false); queue_updating_task->scheduleAfter(QUEUE_UPDATE_ERROR_SLEEP_MS); @@ -4081,7 +3978,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (!to_detached) { Transaction transaction(*this, NO_TRANSACTION_RAW); - renameTempPartAndReplace(part, NO_TRANSACTION_RAW, nullptr, &transaction); + renameTempPartAndReplace(part, transaction); replaced_parts = checkPartChecksumsAndCommit(transaction, part, hardlinked_files); @@ -4340,7 +4237,7 @@ void StorageReplicatedMergeTree::shutdown() /// Ask all parts exchange handlers to finish asap. New ones will fail to start data_parts_exchange_ptr->blocker.cancelForever(); /// Wait for all of them - std::unique_lock lock(data_parts_exchange_ptr->rwlock); + std::lock_guard lock(data_parts_exchange_ptr->rwlock); } } @@ -6602,9 +6499,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom( Transaction transaction(*this, NO_TRANSACTION_RAW); { auto data_parts_lock = lockParts(); - - for (MutableDataPartPtr & part : dst_parts) - renameTempPartAndReplace(part, query_context->getCurrentTransaction().get(), nullptr, &transaction, data_parts_lock); + for (auto & part : dst_parts) + renameTempPartAndReplaceUnlocked(part, transaction, data_parts_lock); } for (size_t i = 0; i < dst_parts.size(); ++i) @@ -6837,11 +6733,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta auto src_data_parts_lock = lockParts(); auto dest_data_parts_lock = dest_table_storage->lockParts(); - std::mutex mutex; - DataPartsLock lock(mutex); - - for (MutableDataPartPtr & part : dst_parts) - dest_table_storage->renameTempPartAndReplace(part, query_context->getCurrentTransaction().get(), nullptr, &transaction, lock); + for (auto & part : dst_parts) + dest_table_storage->renameTempPartAndReplaceUnlocked(part, transaction, dest_data_parts_lock); for (size_t i = 0; i < dst_parts.size(); ++i) dest_table_storage->lockSharedData(*dst_parts[i], false, hardlinked_files_for_parts[i]); @@ -6852,8 +6745,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta else zkutil::KeeperMultiException::check(code, ops, op_results); - parts_to_remove = removePartsInRangeFromWorkingSetAndGetPartsToRemoveFromZooKeeper(NO_TRANSACTION_RAW, drop_range, lock); - transaction.commit(&lock); + parts_to_remove = removePartsInRangeFromWorkingSetAndGetPartsToRemoveFromZooKeeper(NO_TRANSACTION_RAW, drop_range, src_data_parts_lock); + transaction.commit(&src_data_parts_lock); } PartLog::addNewParts(getContext(), dst_parts, watch.elapsed()); @@ -7403,7 +7296,7 @@ void StorageReplicatedMergeTree::checkBrokenDisks() if (disk_ptr->isBroken()) { { - std::unique_lock lock(last_broken_disks_mutex); + std::lock_guard lock(last_broken_disks_mutex); if (!last_broken_disks.insert(disk_ptr->getName()).second) continue; } @@ -7423,7 +7316,7 @@ void StorageReplicatedMergeTree::checkBrokenDisks() else { { - std::unique_lock lock(last_broken_disks_mutex); + std::lock_guard lock(last_broken_disks_mutex); if (last_broken_disks.erase(disk_ptr->getName()) > 0) LOG_INFO( log, @@ -7504,6 +7397,24 @@ void StorageReplicatedMergeTree::createTableSharedID() } +std::optional StorageReplicatedMergeTree::tryGetTableSharedIDFromCreateQuery(const IAST & create_query, const ContextPtr & global_context) +{ + auto zk_path = tryExtractZkPathFromCreateQuery(create_query, global_context); + if (!zk_path) + return {}; + + String zk_name = zkutil::extractZooKeeperName(*zk_path); + zk_path = zkutil::extractZooKeeperPath(*zk_path, false, nullptr); + zkutil::ZooKeeperPtr zookeeper = (zk_name == getDefaultZooKeeperName()) ? global_context->getZooKeeper() : global_context->getAuxiliaryZooKeeper(zk_name); + + String id; + if (!zookeeper->tryGet(fs::path(*zk_path) / "table_shared_id", id)) + return {}; + + return id; +} + + void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const { auto settings = getSettings(); @@ -8020,7 +7931,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP try { MergeTreeData::Transaction transaction(*this, NO_TRANSACTION_RAW); - auto replaced_parts = renameTempPartAndReplace(new_data_part, NO_TRANSACTION_RAW, nullptr, &transaction); + auto replaced_parts = renameTempPartAndReplace(new_data_part, transaction); if (!replaced_parts.empty()) { @@ -8254,44 +8165,21 @@ void StorageReplicatedMergeTree::createAndStoreFreezeMetadata(DiskPtr disk, Data } -ASTPtr StorageReplicatedMergeTree::getCreateQueryForBackup(const ContextPtr & local_context, DatabasePtr * database) const +void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_query) const { - ASTPtr query = MergeTreeData::getCreateQueryForBackup(local_context, database); + /// Adjust the create query using values from ZooKeeper. + auto zookeeper = getZooKeeper(); + auto columns_from_entry = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_path) / "columns")); + auto metadata_from_entry = ReplicatedMergeTreeTableMetadata::parse(zookeeper->get(fs::path(zookeeper_path) / "metadata")); - /// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there - /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. - /// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table. - auto & create = query->as(); - if (create.storage && create.storage->engine && (create.uuid != UUIDHelpers::Nil)) - { - auto & engine = *(create.storage->engine); - if (auto * engine_args_ast = typeid_cast(engine.arguments.get())) - { - auto & engine_args = engine_args_ast->children; - if (engine_args.size() >= 2) - { - auto * zookeeper_path_ast = typeid_cast(engine_args[0].get()); - auto * replica_name_ast = typeid_cast(engine_args[1].get()); - if (zookeeper_path_ast && (zookeeper_path_ast->value.getType() == Field::Types::String) && - replica_name_ast && (replica_name_ast->value.getType() == Field::Types::String)) - { - String & zookeeper_path_arg = zookeeper_path_ast->value.get(); - String & replica_name_arg = replica_name_ast->value.get(); - String table_uuid_str = toString(create.uuid); - if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) - zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); - const auto & config = getContext()->getConfigRef(); - if ((zookeeper_path_arg == getDefaultZooKeeperPath(config)) && (replica_name_arg == getDefaultReplicaName(config)) - && ((engine_args.size() == 2) || !engine_args[2]->as())) - { - engine_args.erase(engine_args.begin(), engine_args.begin() + 2); - } - } - } - } - } + auto current_metadata = getInMemoryMetadataPtr(); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, current_metadata).checkAndFindDiff(metadata_from_entry, current_metadata->getColumns(), getContext()); + auto adjusted_metadata = metadata_diff.getNewMetadata(columns_from_entry, getContext(), *current_metadata); + applyMetadataChangesToCreateQuery(create_query, adjusted_metadata); - return query; + /// Check that tryGetTableSharedIDFromCreateQuery() works for this storage. + if (tryGetTableSharedIDFromCreateQuery(*create_query, getContext()) != getTableSharedID()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Table {} has its shared ID to be different from one from the create query"); } void StorageReplicatedMergeTree::backupData( @@ -8303,8 +8191,8 @@ void StorageReplicatedMergeTree::backupData( auto backup_entries = backupParts(backup_entries_collector.getContext(), "", partitions); auto coordination = backup_entries_collector.getBackupCoordination(); - String full_zk_path = getZooKeeperName() + getZooKeeperPath(); - coordination->addReplicatedDataPath(full_zk_path, data_path_in_backup); + String shared_id = getTableSharedID(); + coordination->addReplicatedDataPath(shared_id, data_path_in_backup); std::unordered_map part_names_with_hashes_calculating; for (auto & [relative_path, backup_entry] : backup_entries) @@ -8342,23 +8230,23 @@ void StorageReplicatedMergeTree::backupData( } /// Send our list of part names to the coordination (to compare with other replicas). - coordination->addReplicatedPartNames(full_zk_path, getStorageID().getFullTableName(), getReplicaName(), part_names_with_hashes); + coordination->addReplicatedPartNames(shared_id, getStorageID().getFullTableName(), getReplicaName(), part_names_with_hashes); /// This task will be executed after all replicas have collected their parts and the coordination is ready to /// give us the final list of parts to add to the BackupEntriesCollector. - auto post_collecting_task = [full_zk_path, + auto post_collecting_task = [shared_id, replica_name = getReplicaName(), coordination, backup_entries = std::move(backup_entries), &backup_entries_collector]() { - Strings data_paths = coordination->getReplicatedDataPaths(full_zk_path); + Strings data_paths = coordination->getReplicatedDataPaths(shared_id); std::vector data_paths_fs; data_paths_fs.reserve(data_paths.size()); for (const auto & data_path : data_paths) data_paths_fs.push_back(data_path); - Strings part_names = coordination->getReplicatedPartNames(full_zk_path, replica_name); + Strings part_names = coordination->getReplicatedPartNames(shared_id, replica_name); std::unordered_set part_names_set{part_names.begin(), part_names.end()}; for (const auto & [relative_path, backup_entry] : backup_entries) @@ -8371,7 +8259,7 @@ void StorageReplicatedMergeTree::backupData( backup_entries_collector.addBackupEntry(data_path / relative_path, backup_entry); } }; - backup_entries_collector.addPostCollectingTask(post_collecting_task); + backup_entries_collector.addPostTask(post_collecting_task); } void StorageReplicatedMergeTree::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 73a08a2b921..18b9ef54777 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -232,8 +232,8 @@ public: int getMetadataVersion() const { return metadata_version; } - /// Returns a slightly changed version of the CREATE TABLE query which must be written to a backup. - ASTPtr getCreateQueryForBackup(const ContextPtr & context, DatabasePtr * database) const override; + /// Modify a CREATE TABLE query to make a variant which must be written to a backup. + void adjustCreateQueryForBackup(ASTPtr & create_query) const override; /// Makes backup entries to backup the data of the storage. void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; @@ -312,6 +312,9 @@ public: // Return table id, common for different replicas String getTableSharedID() const override; + /// Returns the same as getTableSharedID(), but extracts it from a create query. + static std::optional tryGetTableSharedIDFromCreateQuery(const IAST & create_query, const ContextPtr & global_context); + static String getDefaultZooKeeperName() { return default_zookeeper_name; } /// Check if there are new broken disks and enqueue part recovery tasks. diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index d569a81c4a7..2033d33a33d 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -55,6 +55,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; extern const int TIMEOUT_EXCEEDED; + extern const int CANNOT_RESTORE_TABLE; } @@ -527,11 +528,8 @@ std::optional StorageStripeLog::totalBytes(const Settings &) const } -void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) +void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - auto lock_timeout = getLockTimeout(backup_entries_collector.getContext()); loadIndices(lock_timeout); @@ -589,13 +587,13 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec data_path_in_backup_fs / "count.txt", std::make_unique(toString(num_rows))); } -void StorageStripeLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) +void StorageStripeLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto backup = restorer.getBackup(); - if (!restorer.isNonEmptyTableAllowed() && total_bytes && backup->hasFiles(data_path_in_backup)) + if (!backup->hasFiles(data_path_in_backup)) + return; + + if (!restorer.isNonEmptyTableAllowed() && total_bytes) RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); auto lock_timeout = getLockTimeout(restorer.getContext()); @@ -624,6 +622,11 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & auto old_data_size = file_checker.getFileSize(data_file_path); { String file_path_in_backup = data_path_in_backup_fs / fileName(data_file_path); + if (!backup->fileExists(file_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), file_path_in_backup); + } auto backup_entry = backup->readFile(file_path_in_backup); auto in = backup_entry->getReadBuffer(); auto out = disk->writeFile(data_file_path, max_compress_block_size, WriteMode::Append); @@ -634,6 +637,11 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & { String index_path_in_backup = data_path_in_backup_fs / fileName(index_file_path); IndexForNativeFormat extra_indices; + if (!backup->fileExists(index_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), index_path_in_backup); + } auto backup_entry = backup->readFile(index_path_in_backup); auto index_in = backup_entry->getReadBuffer(); CompressedReadBuffer index_compressed_in{*index_in}; diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index b28082325db..56c96a0857c 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -40,7 +40,7 @@ const char * auto_config_build[] "USE_SSL", "@USE_SSL@", "OPENSSL_VERSION", "@OPENSSL_VERSION@", "OPENSSL_IS_BORING_SSL", "@OPENSSL_IS_BORING_SSL@", - "USE_HYPERSCAN", "@ENABLE_HYPERSCAN@", + "USE_VECTORSCAN", "@ENABLE_VECTORSCAN@", "USE_SIMDJSON", "@USE_SIMDJSON@", "USE_ODBC", "@USE_ODBC@", "USE_GRPC", "@USE_GRPC@", diff --git a/src/Storages/System/StorageSystemFilesystemCache.cpp b/src/Storages/System/StorageSystemFilesystemCache.cpp index 4b76163363a..2baddadec90 100644 --- a/src/Storages/System/StorageSystemFilesystemCache.cpp +++ b/src/Storages/System/StorageSystemFilesystemCache.cpp @@ -38,7 +38,7 @@ void StorageSystemFilesystemCache::fillData(MutableColumns & res_columns, Contex for (const auto & [cache_base_path, cache_data] : caches) { - const auto & cache = cache_data.cache; + const auto & cache = cache_data->cache; auto file_segments = cache->getSnapshot(); for (const auto & file_segment : file_segments) diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index b3f1231bd1a..e2bc699d3f1 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -9,6 +9,16 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + namespace DB { @@ -20,6 +30,11 @@ enum class FunctionOrigin : Int8 EXECUTABLE_USER_DEFINED = 2 }; +namespace ErrorCodes +{ + extern const int CANNOT_RESTORE_TABLE; +} + namespace { template @@ -99,4 +114,66 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c fillRow(res_columns, function_name, UInt64(0), "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); } } + +void StorageSystemFunctions::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) +{ + const auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); + const auto & user_defined_sql_functions_names = user_defined_sql_functions_factory.getAllRegisteredNames(); + fs::path data_path_in_backup_fs{data_path_in_backup}; + for (const auto & function_name : user_defined_sql_functions_names) + { + auto ast = user_defined_sql_functions_factory.tryGet(function_name); + if (!ast) + continue; + backup_entries_collector.addBackupEntry( + data_path_in_backup_fs / (escapeForFileName(function_name) + ".sql"), + std::make_shared(queryToString(ast))); + } +} + +void StorageSystemFunctions::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) +{ + auto backup = restorer.getBackup(); + fs::path data_path_in_backup_fs{data_path_in_backup}; + + Strings filenames = backup->listFiles(data_path_in_backup); + for (const auto & filename : filenames) + { + if (!filename.ends_with(".sql")) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File name {} doesn't have the extension .sql", + getStorageID().getFullTableName(), String{data_path_in_backup_fs / filename}); + } + } + + auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); + const auto & restore_settings = restorer.getRestoreSettings(); + auto context = restorer.getContext(); + + for (const auto & filename : filenames) + { + String escaped_function_name = filename.substr(0, filename.length() - strlen(".sql")); + String function_name = unescapeForFileName(escaped_function_name); + + String filepath = data_path_in_backup_fs / filename; + auto function_def_entry = backup->readFile(filepath); + auto function_def_in = function_def_entry->getReadBuffer(); + String function_def; + readStringUntilEOF(function_def, *function_def_in); + + ParserCreateFunctionQuery parser; + ASTPtr ast = parseQuery( + parser, + function_def.data(), + function_def.data() + function_def.size(), + "in file " + filepath + " from backup " + backup->getName(), + 0, + context->getSettingsRef().max_parser_depth); + + bool replace = (restore_settings.create_function == RestoreUDFCreationMode::kReplace); + bool if_not_exists = (restore_settings.create_function == RestoreUDFCreationMode::kCreateIfNotExists); + user_defined_sql_functions_factory.registerFunction(context, function_name, ast, replace, if_not_exists, true); + } +} + } diff --git a/src/Storages/System/StorageSystemFunctions.h b/src/Storages/System/StorageSystemFunctions.h index fdbe79e29a2..606694a4c0b 100644 --- a/src/Storages/System/StorageSystemFunctions.h +++ b/src/Storages/System/StorageSystemFunctions.h @@ -19,6 +19,9 @@ public: static NamesAndTypesList getNamesAndTypes(); + void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; + void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index efe6b93fe57..046db151684 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -122,23 +122,17 @@ void StorageSystemQuotas::fillData(MutableColumns & res_columns, ContextPtr cont } void StorageSystemQuotas::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::QUOTA, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::QUOTA); } void StorageSystemQuotas::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index a009f9d25c9..d39a0d2482d 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -68,11 +68,11 @@ Pipe StorageSystemRemoteDataPaths::read( col_base_path->insert(disk->getPath()); col_cache_base_path->insert(cache_base_path); col_local_path->insert(local_path); - col_remote_path->insert(remote_path); + col_remote_path->insert(remote_path.path); if (cache) { - auto cache_paths = cache->tryGetCachePaths(cache->hash(remote_path)); + auto cache_paths = cache->tryGetCachePaths(cache->hash(remote_path.path)); col_cache_paths->insert(Array(cache_paths.begin(), cache_paths.end())); } else diff --git a/src/Storages/System/StorageSystemRoles.cpp b/src/Storages/System/StorageSystemRoles.cpp index ff3490ce8ba..e5b8d53ce7e 100644 --- a/src/Storages/System/StorageSystemRoles.cpp +++ b/src/Storages/System/StorageSystemRoles.cpp @@ -60,23 +60,17 @@ void StorageSystemRoles::fillData(MutableColumns & res_columns, ContextPtr conte } void StorageSystemRoles::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::ROLE, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::ROLE); } void StorageSystemRoles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 680f90adff7..064f610730d 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -140,23 +140,17 @@ void StorageSystemRowPolicies::fillData(MutableColumns & res_columns, ContextPtr } void StorageSystemRowPolicies::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::ROW_POLICY, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::ROW_POLICY); } void StorageSystemRowPolicies::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp index 7c3ccfe863a..d03848ba68b 100644 --- a/src/Storages/System/StorageSystemSettingsProfiles.cpp +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -87,23 +87,17 @@ void StorageSystemSettingsProfiles::fillData(MutableColumns & res_columns, Conte } void StorageSystemSettingsProfiles::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::SETTINGS_PROFILE, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::SETTINGS_PROFILE); } void StorageSystemSettingsProfiles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index f2cae638d45..be56abfa3e8 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -215,23 +215,17 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte } void StorageSystemUsers::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::USER, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::USER); } void StorageSystemUsers::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/attachSystemTablesImpl.h b/src/Storages/System/attachSystemTablesImpl.h index fcc1ab43a64..a1fae985d92 100644 --- a/src/Storages/System/attachSystemTablesImpl.h +++ b/src/Storages/System/attachSystemTablesImpl.h @@ -22,6 +22,7 @@ void attach(ContextPtr context, IDatabase & system_database, const String & tabl /// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages /// and path is actually not used auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4()); + DatabaseCatalog::instance().addUUIDMapping(table_id.uuid); String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid); system_database.attachTable(context, table_name, std::make_shared(table_id, std::forward(args)...), path); } diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index cfb19869074..76833b3e19a 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1017,7 +1017,7 @@ void StorageWindowView::threadFuncFireProc() if (shutdown_called) return; - std::unique_lock lock(fire_signal_mutex); + std::lock_guard lock(fire_signal_mutex); UInt32 timestamp_now = std::time(nullptr); while (next_fire_signal <= timestamp_now) @@ -1049,7 +1049,7 @@ void StorageWindowView::threadFuncFireProc() void StorageWindowView::threadFuncFireEvent() { - std::unique_lock lock(fire_signal_mutex); + std::lock_guard lock(fire_signal_mutex); LOG_TRACE(log, "Fire events: {}", fire_signal.size()); @@ -1606,7 +1606,7 @@ void StorageWindowView::drop() dropInnerTableIfAny(true, getContext()); } -void StorageWindowView::dropInnerTableIfAny(bool no_delay, ContextPtr local_context) +void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context) { if (!std::exchange(has_inner_table, false)) return; @@ -1614,10 +1614,10 @@ void StorageWindowView::dropInnerTableIfAny(bool no_delay, ContextPtr local_cont try { InterpreterDropQuery::executeDropQuery( - ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, no_delay); + ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, sync); if (has_inner_target_table) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, no_delay); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync); } catch (...) { diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index 86cc80ee8ea..96c034b9590 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -120,7 +120,7 @@ public: void checkTableCanBeDropped() const override; - void dropInnerTableIfAny(bool no_delay, ContextPtr context) override; + void dropInnerTableIfAny(bool sync, ContextPtr context) override; void drop() override; diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 519307ba28a..45e45b505d4 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -55,6 +55,9 @@ endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() +if (TARGET ch_contrib::base-x) + set(USE_BASEX 1) +endif() if (TARGET ch_contrib::yaml_cpp) set(USE_YAML_CPP 1) endif() diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 1ba91a38a60..dbf5adfe174 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -151,6 +151,8 @@ def main(): needs_data = json.load(file_handler) required_builds = len(needs_data) + logging.info("The next builds are required: %s", ", ".join(needs_data)) + gh = Github(get_best_robot_token()) pr_info = PRInfo() rerun_helper = RerunHelper(gh, pr_info, build_check_name) @@ -159,7 +161,6 @@ def main(): sys.exit(0) builds_for_check = CI_CONFIG["builds_report_config"][build_check_name] - logging.info("My reports list %s", builds_for_check) required_builds = required_builds or len(builds_for_check) # Collect reports from json artifacts diff --git a/tests/ci/cherry_pick_utils/cherrypick.py b/tests/ci/cherry_pick_utils/cherrypick.py index 92c87800828..c844beaee88 100644 --- a/tests/ci/cherry_pick_utils/cherrypick.py +++ b/tests/ci/cherry_pick_utils/cherrypick.py @@ -165,7 +165,7 @@ class CherryPick: "user.name=robot-clickhouse", ] - title = (self._pr["title"].replace('"', r"\""),) + title = self._pr["title"].replace('"', r"\"") pr_title = f"Backport #{self._pr['number']} to {self.target_branch}: {title}" self._run(git_prefix + ["checkout", "-f", self.backport_branch]) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index cf0dfe51e9b..9fda2d09ae6 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -225,7 +225,7 @@ if __name__ == "__main__": output_path_log = os.path.join(result_path, "main_script_log.txt") runner_path = os.path.join(repo_path, "tests/integration", "ci-runner.py") - run_command = f"sudo -E {runner_path} | tee {output_path_log}" + run_command = f"sudo -E {runner_path}" logging.info("Going to run command: `%s`", run_command) logging.info( "ENV parameters for runner:\n%s", diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 3e0d4e822b4..f21d1734029 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -5,6 +5,7 @@ # pylint: disable=too-many-lines import enum +from queue import Full import shutil import sys import os @@ -1489,9 +1490,9 @@ def collect_build_flags(args): result.append(BuildFlags.RELEASE) value = clickhouse_execute( - args, "SELECT value FROM system.settings WHERE name = 'default_database_engine'" + args, "SELECT value FROM system.settings WHERE name = 'allow_deprecated_database_ordinary'" ) - if value == b"Ordinary": + if value == b"1": result.append(BuildFlags.ORDINARY_DATABASE) value = int( @@ -1581,15 +1582,20 @@ def do_run_tests(jobs, test_suite: TestSuite, parallel): for _ in range(jobs): parallel_tests_array.append((None, batch_size, test_suite)) - with closing(multiprocessing.Pool(processes=jobs)) as pool: - pool.map_async(run_tests_array, parallel_tests_array) + try: + with closing(multiprocessing.Pool(processes=jobs)) as pool: + pool.map_async(run_tests_array, parallel_tests_array) - for suit in test_suite.parallel_tests: - queue.put(suit, timeout=args.timeout * 1.1) + for suit in test_suite.parallel_tests: + queue.put(suit, timeout=args.timeout * 1.1) - for _ in range(jobs): - queue.put(None, timeout=args.timeout * 1.1) + for _ in range(jobs): + queue.put(None, timeout=args.timeout * 1.1) + queue.close() + except Full: + print("Couldn't put test to the queue within timeout. Server probably hung.") + print_stacktraces() queue.close() pool.join() @@ -2172,4 +2178,7 @@ if __name__ == "__main__": if args.jobs is None: args.jobs = multiprocessing.cpu_count() + if args.db_engine and args.db_engine == "Ordinary": + MESSAGES_TO_RETRY.append(" locking attempt on ") + main(args) diff --git a/tests/config/config.d/database_atomic.xml b/tests/config/config.d/database_atomic.xml index b3f51d51a79..a551e710ca3 100644 --- a/tests/config/config.d/database_atomic.xml +++ b/tests/config/config.d/database_atomic.xml @@ -1,3 +1,8 @@ 60 + + + 0 + 5 + 10 diff --git a/tests/config/users.d/database_ordinary.xml b/tests/config/users.d/database_ordinary.xml index b3b81ee25ff..8ffd2f27a62 100644 --- a/tests/config/users.d/database_ordinary.xml +++ b/tests/config/users.d/database_ordinary.xml @@ -1,7 +1,7 @@ - Ordinary + 1 diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index e4bd1be9027..562497fe8b0 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -367,7 +367,7 @@ class ClickhouseIntegrationTestsRunner: def _get_all_tests(self, repo_path): image_cmd = self._get_runner_image_cmd(repo_path) out_file = "all_tests.txt" - out_file_full = "all_tests_full.txt" + out_file_full = os.path.join(self.result_path, "runner_get_all_tests.log") cmd = ( "cd {repo_path}/tests/integration && " "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " @@ -393,21 +393,16 @@ class ClickhouseIntegrationTestsRunner: not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0 ): - all_tests_full_file_path = ( - "{repo_path}/tests/integration/{out_file}".format( - repo_path=repo_path, out_file=out_file_full - ) - ) - if os.path.isfile(all_tests_full_file_path): + if os.path.isfile(out_file_full): # log runner output logging.info("runner output:") - with open(all_tests_full_file_path, "r") as all_tests_full_file: + with open(out_file_full, "r") as all_tests_full_file: for line in all_tests_full_file: line = line.rstrip() if line: logging.info("runner output: %s", line) else: - logging.info("runner output '%s' is empty", all_tests_full_file_path) + logging.info("runner output '%s' is empty", out_file_full) raise Exception( "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format( diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index e917e5a1885..0d32547358c 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3390,14 +3390,6 @@ class ClickHouseInstance: ], user="root", ) - self.exec_in_container( - [ - "bash", - "-c", - "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse", - ], - user="root", - ) self.exec_in_container( ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()), @@ -3411,7 +3403,11 @@ class ClickHouseInstance: self.wait_start(time_left) def restart_with_latest_version( - self, stop_start_wait_sec=300, callback_onstop=None, signal=15 + self, + stop_start_wait_sec=300, + callback_onstop=None, + signal=15, + fix_metadata=False, ): begin_time = time.time() if not self.stay_alive: @@ -3458,14 +3454,23 @@ class ClickHouseInstance: "echo 'restart_with_latest_version: From version' && /usr/share/clickhouse_original server --version && echo 'To version' /usr/share/clickhouse_fresh server --version", ] ) - self.exec_in_container( - [ - "bash", - "-c", - "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse", - ], - user="root", - ) + if fix_metadata: + # Versions older than 20.7 might not create .sql file for system and default database + # Create it manually if upgrading from older version + self.exec_in_container( + [ + "bash", + "-c", + "echo 'ATTACH DATABASE system ENGINE=Ordinary' > /var/lib/clickhouse/metadata/system.sql", + ], + ) + self.exec_in_container( + [ + "bash", + "-c", + "echo 'ATTACH DATABASE system ENGINE=Ordinary' > /var/lib/clickhouse/metadata/default.sql", + ], + ) self.exec_in_container( ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()), diff --git a/tests/integration/runner b/tests/integration/runner index 7a02ec309a0..cd07875ad1d 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -25,7 +25,7 @@ VOLUME_NAME = "clickhouse_integration_tests" CONTAINER_NAME = f"{VOLUME_NAME}_{random_str()}" CONFIG_DIR_IN_REPO = "programs/server" -INTERGATION_DIR_IN_REPO = "tests/integration" +INTEGRATION_DIR_IN_REPO = "tests/integration" SRC_DIR_IN_REPO = "src" DIND_INTEGRATION_TESTS_IMAGE_NAME = "clickhouse/integration-tests-runner" @@ -84,7 +84,7 @@ def check_args_and_update_paths(args): ) else: args.cases_dir = os.path.abspath( - os.path.join(CLICKHOUSE_ROOT, INTERGATION_DIR_IN_REPO) + os.path.join(CLICKHOUSE_ROOT, INTEGRATION_DIR_IN_REPO) ) logging.info("Cases dir is not set. Will use %s" % (args.cases_dir)) @@ -346,6 +346,17 @@ if __name__ == "__main__": ) except Exception as ex: print("Volume creationg failed, probably it already exists, exception", ex) + # TODO: this part cleans out stale volumes produced by container name + # randomizer, we should remove it after Sep 2022 + try: + subprocess.check_call( + "docker volume rm $(docker volume ls -q | " + f"grep '{VOLUME_NAME}_.*_volume')", + shell=True, + ) + except Exception as ex: + print("Probably, some stale volumes still there, just continue:", ex) + # TODO END dockerd_internal_volume = f"--volume={VOLUME_NAME}_volume:/var/lib/docker" # If enabled we kill and remove containers before pytest session run. @@ -392,15 +403,15 @@ if __name__ == "__main__": command=args.command, ) - try: - print("Trying to kill container", CONTAINER_NAME, "if it's already running") - subprocess.check_call( - f'docker kill $(docker ps -a -q --filter name={CONTAINER_NAME} --format="{{{{.ID}}}}")', - shell=True, - ) - print("Container killed") - except: - print("Nothing to kill") + containers = subprocess.check_output( + f"docker ps -a -q --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", + shell=True, + universal_newlines=True, + ).splitlines() + if containers: + print(f"Trying to kill containers name={CONTAINER_NAME} ids={containers}") + subprocess.check_call(f"docker kill {' '.join(containers)}", shell=True) + print(f"Containers {containers} killed") print(("Running pytest container as: '" + cmd + "'.")) subprocess.check_call(cmd, shell=True) diff --git a/tests/integration/test_atomic_drop_table/test.py b/tests/integration/test_atomic_drop_table/test.py index 1fe88dde099..6ffa60de7b5 100644 --- a/tests/integration/test_atomic_drop_table/test.py +++ b/tests/integration/test_atomic_drop_table/test.py @@ -20,7 +20,8 @@ def start_cluster(): try: cluster.start() node1.query( - "CREATE DATABASE zktest ENGINE=Ordinary;" + "CREATE DATABASE zktest ENGINE=Ordinary;", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different behaviour with Atomic node1.query( """ diff --git a/tests/integration/test_attach_partition_with_large_destination/test.py b/tests/integration/test_attach_partition_with_large_destination/test.py index 0a4ab9fada1..a82e63bb7ba 100644 --- a/tests/integration/test_attach_partition_with_large_destination/test.py +++ b/tests/integration/test_attach_partition_with_large_destination/test.py @@ -34,7 +34,10 @@ def create_force_drop_flag(node): @pytest.mark.parametrize("engine", ["Ordinary", "Atomic"]) def test_attach_partition_with_large_destination(started_cluster, engine): # Initialize - node.query("CREATE DATABASE db ENGINE={}".format(engine)) + node.query( + "CREATE DATABASE db ENGINE={}".format(engine), + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query( "CREATE TABLE db.destination (n UInt64) ENGINE=ReplicatedMergeTree('/test/destination', 'r1') ORDER BY n PARTITION BY n % 2" ) diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py index 905abef05b0..193e638186c 100644 --- a/tests/integration/test_backup_restore/test.py +++ b/tests/integration/test_backup_restore/test.py @@ -15,7 +15,8 @@ def started_cluster(): try: cluster.start() instance.query( - "CREATE DATABASE test ENGINE = Ordinary" + "CREATE DATABASE test ENGINE = Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different path in shadow/ with Atomic instance.query("DROP TABLE IF EXISTS test.tbl") instance.query( diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 3996a31e7c9..a930ddac7df 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -138,7 +138,7 @@ def test_backup_table_under_another_name(): assert instance.query("SELECT count(), sum(x) FROM test.table2") == "100\t4950\n" -def test_materialized_view(): +def test_materialized_view_select_1(): backup_name = new_backup_name() instance.query( "CREATE MATERIALIZED VIEW mv_1(x UInt8) ENGINE=MergeTree ORDER BY tuple() POPULATE AS SELECT 1 AS x" @@ -456,18 +456,32 @@ def test_temporary_table(): ) == TSV([["e"], ["q"], ["w"]]) -# "BACKUP DATABASE _temporary_and_external_tables" is allowed but the backup must not contain these tables. -def test_temporary_tables_database(): +# The backup created by "BACKUP DATABASE _temporary_and_external_tables" must not contain tables from other sessions. +def test_temporary_database(): session_id = new_session_id() instance.http_query( "CREATE TEMPORARY TABLE temp_tbl(s String)", params={"session_id": session_id} ) - backup_name = new_backup_name() - instance.query(f"BACKUP DATABASE _temporary_and_external_tables TO {backup_name}") + other_session_id = new_session_id() + instance.http_query( + "CREATE TEMPORARY TABLE other_temp_tbl(s String)", + params={"session_id": other_session_id}, + ) - assert os.listdir(os.path.join(get_path_to_backup(backup_name), "metadata/")) == [ - "_temporary_and_external_tables.sql" # database metadata only + backup_name = new_backup_name() + instance.http_query( + f"BACKUP DATABASE _temporary_and_external_tables TO {backup_name}", + params={"session_id": session_id}, + ) + + assert os.listdir( + os.path.join(get_path_to_backup(backup_name), "temporary_tables/metadata") + ) == ["temp_tbl.sql"] + + assert sorted(os.listdir(get_path_to_backup(backup_name))) == [ + ".backup", + "temporary_tables", ] @@ -711,3 +725,107 @@ def test_system_users_async(): instance.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_c = 3\n" ) + + +def test_projection(): + create_and_fill_table(n=3) + + instance.query("ALTER TABLE test.table ADD PROJECTION prjmax (SELECT MAX(x))") + instance.query(f"INSERT INTO test.table VALUES (100, 'a'), (101, 'b')") + + assert ( + instance.query( + "SELECT count() FROM system.projection_parts WHERE database='test' AND table='table' AND name='prjmax'" + ) + == "2\n" + ) + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + + assert os.path.exists( + os.path.join( + get_path_to_backup(backup_name), "data/test/table/1_5_5_0/data.bin" + ) + ) + + assert os.path.exists( + os.path.join( + get_path_to_backup(backup_name), + "data/test/table/1_5_5_0/prjmax.proj/data.bin", + ) + ) + + instance.query("DROP TABLE test.table") + + assert ( + instance.query( + "SELECT count() FROM system.projection_parts WHERE database='test' AND table='table' AND name='prjmax'" + ) + == "0\n" + ) + + instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV( + [[0, "0"], [1, "1"], [2, "2"], [100, "a"], [101, "b"]] + ) + + assert ( + instance.query( + "SELECT count() FROM system.projection_parts WHERE database='test' AND table='table' AND name='prjmax'" + ) + == "2\n" + ) + + +def test_system_functions(): + instance.query("CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b;") + + instance.query("CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even');") + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE system.functions TO {backup_name}") + + instance.query("DROP FUNCTION linear_equation") + instance.query("DROP FUNCTION parity_str") + + instance.query(f"RESTORE TABLE system.functions FROM {backup_name}") + + assert instance.query( + "SELECT number, linear_equation(number, 2, 1) FROM numbers(3)" + ) == TSV([[0, 1], [1, 3], [2, 5]]) + + assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV( + [[0, "even"], [1, "odd"], [2, "even"]] + ) + + +def test_backup_partition(): + create_and_fill_table(n=30) + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE test.table PARTITIONS '1', '4' TO {backup_name}") + + instance.query("DROP TABLE test.table") + + instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV( + [[1, "1"], [4, "4"], [11, "11"], [14, "14"], [21, "21"], [24, "24"]] + ) + + +def test_restore_partition(): + create_and_fill_table(n=30) + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + + instance.query("DROP TABLE test.table") + + instance.query(f"RESTORE TABLE test.table PARTITIONS '2', '3' FROM {backup_name}") + + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV( + [[2, "2"], [3, "3"], [12, "12"], [13, "13"], [22, "22"], [23, "23"]] + ) diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 6264959fbce..02f855cf766 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -514,3 +514,141 @@ def test_system_users(): node1.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS custom_a = 123\n" ) assert node1.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON default.tbl TO u1\n" + + +def test_projection(): + node1.query( + "CREATE TABLE tbl ON CLUSTER 'cluster' (x UInt32, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}') " + "ORDER BY y PARTITION BY x%10" + ) + node1.query(f"INSERT INTO tbl SELECT number, toString(number) FROM numbers(3)") + + node1.query("ALTER TABLE tbl ADD PROJECTION prjmax (SELECT MAX(x))") + node1.query(f"INSERT INTO tbl VALUES (100, 'a'), (101, 'b')") + + assert ( + node1.query( + "SELECT count() FROM system.projection_parts WHERE database='default' AND table='tbl' AND name='prjmax'" + ) + == "2\n" + ) + + backup_name = new_backup_name() + node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") + + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + + assert ( + node1.query( + "SELECT count() FROM system.projection_parts WHERE database='default' AND table='tbl' AND name='prjmax'" + ) + == "0\n" + ) + + node1.query(f"RESTORE TABLE tbl FROM {backup_name}") + + assert node1.query("SELECT * FROM tbl ORDER BY x") == TSV( + [[0, "0"], [1, "1"], [2, "2"], [100, "a"], [101, "b"]] + ) + + assert ( + node1.query( + "SELECT count() FROM system.projection_parts WHERE database='default' AND table='tbl' AND name='prjmax'" + ) + == "2\n" + ) + + +def test_replicated_table_with_not_synced_def(): + node1.query( + "CREATE TABLE tbl (" + "x UInt8, y String" + ") ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')" + "ORDER BY tuple()" + ) + + node2.query( + "CREATE TABLE tbl (" + "x UInt8, y String" + ") ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')" + "ORDER BY tuple()" + ) + + node2.query("SYSTEM STOP REPLICATION QUEUES tbl") + node1.query("ALTER TABLE tbl MODIFY COLUMN x String") + + # Not synced because the replication queue is stopped + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "UInt8"], ["y", "String"]]) + + backup_name = new_backup_name() + node2.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") + + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + + # But synced after RESTORE anyway + node1.query( + f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=1" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + + node2.query( + f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + + +def test_table_in_replicated_database_with_not_synced_def(): + node1.query( + "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" + ) + + node1.query( + "CREATE TABLE mydb.tbl (x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY tuple()" + ) + + node1.query("ALTER TABLE mydb.tbl MODIFY COLUMN x String") + + backup_name = new_backup_name() + node2.query(f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name}") + + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + + # But synced after RESTORE anyway + node1.query( + f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=1" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + + node2.query( + f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 9cb998fb505..d30c45c3691 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -54,7 +54,7 @@ def test_backup_from_old_version(started_cluster): node1.query("ALTER TABLE source_table FREEZE PARTITION tuple();") - node1.restart_with_latest_version() + node1.restart_with_latest_version(fix_metadata=True) node1.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()" @@ -107,7 +107,7 @@ def test_backup_from_old_version_setting(started_cluster): node2.query("ALTER TABLE source_table FREEZE PARTITION tuple();") - node2.restart_with_latest_version() + node2.restart_with_latest_version(fix_metadata=True) node2.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" @@ -163,7 +163,7 @@ def test_backup_from_old_version_config(started_cluster): "1", ) - node3.restart_with_latest_version(callback_onstop=callback) + node3.restart_with_latest_version(callback_onstop=callback, fix_metadata=True) node3.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" @@ -202,7 +202,8 @@ def test_backup_from_old_version_config(started_cluster): def test_backup_and_alter(started_cluster): node4.query( - "CREATE DATABASE test ENGINE=Ordinary" + "CREATE DATABASE test ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different path in shadow/ with Atomic node4.query( diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py index b3ad9011239..13dd28ee8af 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py @@ -71,7 +71,7 @@ def test_backward_compatability(start_cluster): assert node1.query("SELECT avgMerge(x) FROM state") == "2.5\n" - node1.restart_with_latest_version() + node1.restart_with_latest_version(fix_metadata=True) assert node1.query("SELECT avgMerge(x) FROM state") == "2.5\n" diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py new file mode 100644 index 00000000000..59ceca23a51 --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -0,0 +1,77 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__, name="convert_ordinary") +node = cluster.add_instance( + "node", + image="yandex/clickhouse-server", + tag="19.17.8.54", + stay_alive=True, + with_installed_binary=True, +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(query): + return node.query(query, settings={"log_queries": 1}) + + +def test_convert_system_db_to_atomic(start_cluster): + q( + "CREATE TABLE t(date Date, id UInt32) ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY id" + ) + q("INSERT INTO t VALUES (today(), 1)") + q("INSERT INTO t SELECT number % 1000, number FROM system.numbers LIMIT 1000000") + + assert "1000001\n" == q("SELECT count() FROM t") + assert "499999500001\n" == q("SELECT sum(id) FROM t") + assert "1970-01-01\t1000\t499500000\n1970-01-02\t1000\t499501000\n" == q( + "SELECT date, count(), sum(id) FROM t GROUP BY date ORDER BY date LIMIT 2" + ) + q("SYSTEM FLUSH LOGS") + + assert "query_log" in q("SHOW TABLES FROM system") + assert "part_log" in q("SHOW TABLES FROM system") + q("SYSTEM FLUSH LOGS") + assert "1\n" == q("SELECT count() != 0 FROM system.query_log") + assert "1\n" == q("SELECT count() != 0 FROM system.part_log") + + node.restart_with_latest_version(fix_metadata=True) + + assert "Ordinary" in node.query("SHOW CREATE DATABASE default") + assert "Atomic" in node.query("SHOW CREATE DATABASE system") + assert "query_log" in node.query("SHOW TABLES FROM system") + assert "part_log" in node.query("SHOW TABLES FROM system") + node.query("SYSTEM FLUSH LOGS") + + assert "query_log_0" in node.query("SHOW TABLES FROM system") + assert "part_log_0" in node.query("SHOW TABLES FROM system") + assert "1\n" == node.query("SELECT count() != 0 FROM system.query_log_0") + assert "1\n" == node.query("SELECT count() != 0 FROM system.part_log_0") + assert "1970-01-01\t1000\t499500000\n1970-01-02\t1000\t499501000\n" == node.query( + "SELECT date, count(), sum(id) FROM t GROUP BY date ORDER BY date LIMIT 2" + ) + assert "INFORMATION_SCHEMA\ndefault\ninformation_schema\nsystem\n" == node.query( + "SELECT name FROM system.databases ORDER BY name" + ) + + errors_count = node.count_in_log("") + assert "0\n" == errors_count or ( + "1\n" == errors_count + and "1\n" == node.count_in_log("Can't receive Netlink response") + ) + assert "0\n" == node.count_in_log(" Database") + errors_count = node.count_in_log("always include the lines below") + assert "0\n" == errors_count or ( + "1\n" == errors_count + and "1\n" == node.count_in_log("Can't receive Netlink response") + ) diff --git a/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml b/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml new file mode 100644 index 00000000000..5fbe87cce00 --- /dev/null +++ b/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml @@ -0,0 +1,11 @@ + + 0 + 60 + 1 + + + + testkeeper + + \ No newline at end of file diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index 3d9134bdc54..019ae3d52a1 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -1,14 +1,15 @@ import pytest from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk from helpers.corrupt_part_data_on_disk import break_part import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", stay_alive=True, with_zookeeper=True) +node1 = cluster.add_instance( + "node1", stay_alive=True, main_configs=["configs/store_cleanup.xml"] +) path_to_data = "/var/lib/clickhouse/" @@ -147,3 +148,184 @@ def test_remove_broken_detached_part_replicated_merge_tree(started_cluster): ) remove_broken_detached_part_impl("replicated_mt", node1, "broken") + + +def test_store_cleanup(started_cluster): + node1.query("CREATE DATABASE db UUID '10000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db.log UUID '10000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query( + "CREATE TABLE db.mt UUID '10000000-1000-4000-8000-000000000003' ENGINE=MergeTree ORDER BY tuple() AS SELECT 1" + ) + node1.query( + "CREATE TABLE db.mem UUID '10000000-1000-4000-8000-000000000004' ENGINE=Memory AS SELECT 1" + ) + + node1.query("CREATE DATABASE db2 UUID '20000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db2.log UUID '20000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query("DETACH DATABASE db2") + + node1.query("CREATE DATABASE db3 UUID '30000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db3.log UUID '30000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query( + "CREATE TABLE db3.log2 UUID '30000000-1000-4000-8000-000000000003' ENGINE=Log AS SELECT 1" + ) + node1.query("DETACH TABLE db3.log") + node1.query("DETACH TABLE db3.log2 PERMANENTLY") + + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + node1.stop_clickhouse(kill=True) + # All dirs related to `db` will be removed + node1.exec_in_container(["rm", f"{path_to_data}/metadata/db.sql"]) + + node1.exec_in_container(["mkdir", f"{path_to_data}/store/kek"]) + node1.exec_in_container(["touch", f"{path_to_data}/store/12"]) + node1.exec_in_container(["mkdir", f"{path_to_data}/store/456"]) + node1.exec_in_container(["mkdir", f"{path_to_data}/store/456/testgarbage"]) + node1.exec_in_container( + ["mkdir", f"{path_to_data}/store/456/30000000-1000-4000-8000-000000000003"] + ) + node1.exec_in_container( + ["touch", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000003"] + ) + node1.exec_in_container( + ["mkdir", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000004"] + ) + + node1.start_clickhouse() + node1.query("DETACH DATABASE db2") + node1.query("DETACH TABLE db3.log") + + node1.wait_for_log_line( + "Removing access rights for unused directory", + timeout=60, + look_behind_lines=1000, + ) + node1.wait_for_log_line("directories from store") + + store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) + assert "100" in store + assert "200" in store + assert "300" in store + assert "456" in store + assert "kek" in store + assert "12" in store + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) + + # Metadata is removed, so store/100 contains garbage + store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"]) + assert "10000000-1000-4000-8000-000000000001" in store100 + assert "10000000-1000-4000-8000-000000000002" in store100 + assert "10000000-1000-4000-8000-000000000003" in store100 + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) + + # Database is detached, nothing to clean up + store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"]) + assert "20000000-1000-4000-8000-000000000001" in store200 + assert "20000000-1000-4000-8000-000000000002" in store200 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + + # Tables are detached, nothing to clean up + store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"]) + assert "30000000-1000-4000-8000-000000000001" in store300 + assert "30000000-1000-4000-8000-000000000002" in store300 + assert "30000000-1000-4000-8000-000000000003" in store300 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + # Manually created garbage + store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"]) + assert "30000000-1000-4000-8000-000000000003" in store456 + assert "45600000-1000-4000-8000-000000000003" in store456 + assert "45600000-1000-4000-8000-000000000004" in store456 + assert "testgarbage" in store456 + assert "----------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) + + node1.wait_for_log_line( + "Removing unused directory", timeout=90, look_behind_lines=1000 + ) + node1.wait_for_log_line("directories from store") + + store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) + assert "100" in store + assert "200" in store + assert "300" in store + assert "456" in store + assert "kek" not in store # changed + assert "\n12\n" not in store # changed + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) # changed + + # Metadata is removed, so store/100 contains garbage + store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"]) # changed + assert "10000000-1000-4000-8000-000000000001" not in store100 # changed + assert "10000000-1000-4000-8000-000000000002" not in store100 # changed + assert "10000000-1000-4000-8000-000000000003" not in store100 # changed + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) # changed + + # Database is detached, nothing to clean up + store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"]) + assert "20000000-1000-4000-8000-000000000001" in store200 + assert "20000000-1000-4000-8000-000000000002" in store200 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + + # Tables are detached, nothing to clean up + store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"]) + assert "30000000-1000-4000-8000-000000000001" in store300 + assert "30000000-1000-4000-8000-000000000002" in store300 + assert "30000000-1000-4000-8000-000000000003" in store300 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + # Manually created garbage + store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"]) + assert "30000000-1000-4000-8000-000000000003" not in store456 # changed + assert "45600000-1000-4000-8000-000000000003" not in store456 # changed + assert "45600000-1000-4000-8000-000000000004" not in store456 # changed + assert "testgarbage" not in store456 # changed + assert "---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) # changed + + node1.query("ATTACH TABLE db3.log2") + node1.query("ATTACH DATABASE db2") + node1.query("ATTACH TABLE db3.log") + + assert "1\n" == node1.query("SELECT * FROM db3.log") + assert "1\n" == node1.query("SELECT * FROM db3.log2") + assert "1\n" == node1.query("SELECT * FROM db2.log") diff --git a/tests/integration/test_cluster_copier/configs/users.xml b/tests/integration/test_cluster_copier/configs/users.xml index 492cf4d7ee6..2542642f6df 100644 --- a/tests/integration/test_cluster_copier/configs/users.xml +++ b/tests/integration/test_cluster_copier/configs/users.xml @@ -5,6 +5,7 @@ 1 5 + 1 diff --git a/tests/integration/test_cluster_copier/configs_three_nodes/users.xml b/tests/integration/test_cluster_copier/configs_three_nodes/users.xml index ce3538a31b8..f017daff974 100644 --- a/tests/integration/test_cluster_copier/configs_three_nodes/users.xml +++ b/tests/integration/test_cluster_copier/configs_three_nodes/users.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_cluster_copier/configs_two_nodes/users.xml b/tests/integration/test_cluster_copier/configs_two_nodes/users.xml index ce3538a31b8..f017daff974 100644 --- a/tests/integration/test_cluster_copier/configs_two_nodes/users.xml +++ b/tests/integration/test_cluster_copier/configs_two_nodes/users.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_config_substitutions/configs/config_allow_databases.xml b/tests/integration/test_config_substitutions/configs/config_allow_databases.xml index 98008306787..be727360dcf 100644 --- a/tests/integration/test_config_substitutions/configs/config_allow_databases.xml +++ b/tests/integration/test_config_substitutions/configs/config_allow_databases.xml @@ -1,4 +1,9 @@ + + + 1 + + diff --git a/tests/integration/test_cross_replication/test.py b/tests/integration/test_cross_replication/test.py index 143b8823bf2..2a73acadafd 100644 --- a/tests/integration/test_cross_replication/test.py +++ b/tests/integration/test_cross_replication/test.py @@ -37,7 +37,7 @@ def started_cluster(): CREATE DATABASE shard_{shard}; CREATE TABLE shard_{shard}.replicated(date Date, id UInt32, shard_id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index 4af276b9728..5d033ac8f7e 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -421,7 +421,7 @@ def test_default_codec_version_update(start_cluster): ) old_version = node3.query("SELECT version()") - node3.restart_with_latest_version() + node3.restart_with_latest_version(fix_metadata=True) new_version = node3.query("SELECT version()") logging.debug(f"Updated from {old_version} to {new_version}") assert ( diff --git a/tests/integration/test_delayed_replica_failover/test.py b/tests/integration/test_delayed_replica_failover/test.py index 387d6a12f48..a480ee3f278 100644 --- a/tests/integration/test_delayed_replica_failover/test.py +++ b/tests/integration/test_delayed_replica_failover/test.py @@ -32,7 +32,7 @@ def started_cluster(): node.query( """ CREATE TABLE replicated (d Date, x UInt32) ENGINE = - ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)""".format( + ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}') PARTITION BY toYYYYMM(d) ORDER BY d""".format( shard=shard, instance=node.name ) ) diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index f57d4e42813..2042db69fa2 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -16,7 +16,10 @@ def start_cluster(): for node in nodes: node.query("CREATE DATABASE IF NOT EXISTS test") # Different internal dictionary name with Atomic - node.query("CREATE DATABASE IF NOT EXISTS test_ordinary ENGINE=Ordinary") + node.query( + "CREATE DATABASE IF NOT EXISTS test_ordinary ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query("CREATE DATABASE IF NOT EXISTS atest") node.query("CREATE DATABASE IF NOT EXISTS ztest") node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log") diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index d950959f3e1..419d0fbe8ca 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -58,6 +58,22 @@ def test_disks_app_func_ls(started_cluster): assert files[0] == "store" + out = source.exec_in_container( + [ + "/usr/bin/clickhouse", + "disks", + "--send-logs", + "--disk", + "test1", + "list", + ".", + "--recursive", + ] + ) + + assert ".:\nstore\n" in out + assert "\n./store:\n" in out + def test_disks_app_func_cp(started_cluster): source = cluster.instances["disks_app_test"] diff --git a/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml b/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml index 26db7f54514..ef8abbd9174 100644 --- a/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml +++ b/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml b/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml index 26db7f54514..ef8abbd9174 100644 --- a/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml +++ b/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 2789541b519..85d0a5f0999 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -552,7 +552,9 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( - instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary" + instance, + "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) assert ( "are supported only for ON CLUSTER queries with Atomic database engine" diff --git a/tests/integration/test_distributed_storage_configuration/test.py b/tests/integration/test_distributed_storage_configuration/test.py index fa4e01bb7b3..950ce1034fe 100644 --- a/tests/integration/test_distributed_storage_configuration/test.py +++ b/tests/integration/test_distributed_storage_configuration/test.py @@ -20,7 +20,8 @@ def start_cluster(): try: cluster.start() node.query( - "CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary" + "CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different paths with Atomic yield cluster finally: diff --git a/tests/integration/test_extreme_deduplication/test.py b/tests/integration/test_extreme_deduplication/test.py index 2c8772aad4e..71f783d37c9 100644 --- a/tests/integration/test_extreme_deduplication/test.py +++ b/tests/integration/test_extreme_deduplication/test.py @@ -40,7 +40,7 @@ def test_deduplication_window_in_seconds(started_cluster): node1.query( """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id""" ) node.query("INSERT INTO simple VALUES (0, 0)") @@ -77,7 +77,7 @@ def test_deduplication_works_in_case_of_intensive_inserts(started_cluster): node1.query( """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id""" ) node1.query("INSERT INTO simple VALUES (0, 0)") diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 34e377e0ae4..898bbc40eb9 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -16,7 +16,10 @@ def started_cluster(): def test_file_path_escaping(started_cluster): - node.query("CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary") + node.query( + "CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query( """ CREATE TABLE test.`T.a_b,l-e!` (`~Id` UInt32) diff --git a/tests/integration/test_force_drop_table/test.py b/tests/integration/test_force_drop_table/test.py index c1eec1cd277..ae6c7a1a3bf 100644 --- a/tests/integration/test_force_drop_table/test.py +++ b/tests/integration/test_force_drop_table/test.py @@ -33,7 +33,10 @@ def create_force_drop_flag(node): @pytest.mark.parametrize("engine", ["Ordinary", "Atomic"]) def test_drop_materialized_view(started_cluster, engine): - node.query("CREATE DATABASE d ENGINE={}".format(engine)) + node.query( + "CREATE DATABASE d ENGINE={}".format(engine), + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query( "CREATE TABLE d.rmt (n UInt64) ENGINE=ReplicatedMergeTree('/test/rmt', 'r1') ORDER BY n PARTITION BY n % 2" ) diff --git a/tests/integration/test_hive_query/data/prepare_hive_data.sh b/tests/integration/test_hive_query/data/prepare_hive_data.sh index 8126b975612..495ea201870 100755 --- a/tests/integration/test_hive_query/data/prepare_hive_data.sh +++ b/tests/integration/test_hive_query/data/prepare_hive_data.sh @@ -1,10 +1,15 @@ #!/bin/bash hive -e "create database test" -hive -e "create table test.demo(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; create table test.demo_orc(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; " -hive -e "create table test.parquet_demo(id string, score int) PARTITIONED BY(day string, hour string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" -hive -e "create table test.demo_text(id string, score int, day string)row format delimited fields terminated by ','; load data local inpath '/demo_data.txt' into table test.demo_text " +hive -e "drop table if exists test.demo; create table test.demo(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; create table test.demo_orc(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; " +hive -e "drop table if exists test.parquet_demo; create table test.parquet_demo(id string, score int) PARTITIONED BY(day string, hour string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" +hive -e "drop table if exists test.demo_text; create table test.demo_text(id string, score int, day string)row format delimited fields terminated by ','; load data local inpath '/demo_data.txt' into table test.demo_text " hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo partition(day) select * from test.demo_text; insert into test.demo_orc partition(day) select * from test.demo_text" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;" + +hive -e "drop table if exists test.test_hive_types; CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct>) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" + +hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 10, 'y', 'xyz')); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 11, 'y', 'abc'));" + diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index a28269f3521..276fd42de9b 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -408,3 +408,38 @@ def test_cache_dir_use(started_cluster): ["bash", "-c", "ls /tmp/clickhouse_local_cache1 | wc -l"] ) assert result0 != "0" and result1 != "0" + + +def test_cache_dir_use(started_cluster): + node = started_cluster.instances["h0_0_0"] + result0 = node.exec_in_container( + ["bash", "-c", "ls /tmp/clickhouse_local_cache | wc -l"] + ) + result1 = node.exec_in_container( + ["bash", "-c", "ls /tmp/clickhouse_local_cache1 | wc -l"] + ) + assert result0 != "0" and result1 != "0" + + +def test_hive_struct_type(started_cluster): + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ + CREATE TABLE IF NOT EXISTS default.test_hive_types (`f_tinyint` Int8, `f_smallint` Int16, `f_int` Int32, `f_integer` Int32, `f_bigint` Int64, `f_float` Float32, `f_double` Float64, `f_decimal` Float64, `f_timestamp` DateTime, `f_date` Date, `f_string` String, `f_varchar` String, `f_char` String, `f_bool` Boolean, `f_array_int` Array(Int32), `f_array_string` Array(String), `f_array_float` Array(Float32), `f_map_int` Map(String, Int32), `f_map_string` Map(String, String), `f_map_float` Map(String, Float32), `f_struct` Tuple(a String, b Int32, c Float32, d Tuple(x Int32, y String)), `day` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'test_hive_types') PARTITION BY (day) + """ + ) + result = node.query( + """ + SELECT * FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 + """ + ) + expected_result = """1 2 3 4 5 6.11 7.22 8 2022-02-20 14:47:04 2022-02-20 hello world hello world hello world true [1,2,3] ['hello world','hello world'] [1.1,1.2] {'a':100,'b':200,'c':300} {'a':'aa','b':'bb','c':'cc'} {'a':111.1,'b':222.2,'c':333.3} ('aaa',200,333.3,(10,'xyz')) 2022-02-20""" + assert result.strip() == expected_result + + result = node.query( + """ + SELECT day, f_struct.a, f_struct.d.x FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 + """ + ) + expected_result = """2022-02-20 aaa 10""" + assert result.strip() == expected_result diff --git a/tests/integration/test_https_replication/test.py b/tests/integration/test_https_replication/test.py index 4cf9f19b870..301487aa6cf 100644 --- a/tests/integration/test_https_replication/test.py +++ b/tests/integration/test_https_replication/test.py @@ -19,7 +19,7 @@ def _fill_nodes(nodes, shard): CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index b8d94d2a043..a52809f817c 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -52,7 +52,7 @@ CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'defaul ) remote.query( - "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)" + "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x" ) instance_test_inserts_batching.query( """ @@ -61,7 +61,7 @@ CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', ) instance_test_inserts_local_cluster.query( - "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)" + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x" ) instance_test_inserts_local_cluster.query( """ @@ -71,12 +71,12 @@ CREATE TABLE distributed_on_local (d Date, x UInt32) ENGINE = Distributed('test_ node1.query( """ -CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node1', date, id, 8192) +CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node1') PARTITION BY toYYYYMM(date) ORDER BY id """ ) node2.query( """ -CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node2', date, id, 8192) +CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node2') PARTITION BY toYYYYMM(date) ORDER BY id """ ) @@ -94,12 +94,12 @@ CREATE TABLE distributed (date Date, id UInt32) ENGINE = Distributed('shard_with shard1.query( """ -CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)""" +CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x""" ) shard2.query( """ -CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)""" +CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x""" ) shard1.query( @@ -143,7 +143,7 @@ CREATE TABLE distributed_one_replica_no_internal_replication (date Date, id UInt node2.query( """ -CREATE TABLE single_replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/single_replicated', 'node2', date, id, 8192) +CREATE TABLE single_replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/single_replicated', 'node2') PARTITION BY toYYYYMM(date) ORDER BY id """ ) @@ -228,11 +228,11 @@ def test_inserts_batching(started_cluster): # 4. Full batch of inserts after ALTER (that have different block structure). # 5. What was left to insert with the column structure before ALTER. expected = """\ -20000101_20000101_1_1_0\t[1] -20000101_20000101_2_2_0\t[2,3,4] -20000101_20000101_3_3_0\t[5,6,7] -20000101_20000101_4_4_0\t[10,11,12] -20000101_20000101_5_5_0\t[8,9] +200001_1_1_0\t[1] +200001_2_2_0\t[2,3,4] +200001_3_3_0\t[5,6,7] +200001_4_4_0\t[10,11,12] +200001_5_5_0\t[8,9] """ assert TSV(result) == TSV(expected) diff --git a/tests/integration/test_insert_into_distributed_sync_async/test.py b/tests/integration/test_insert_into_distributed_sync_async/test.py index e0c454feee6..12423cc4747 100755 --- a/tests/integration/test_insert_into_distributed_sync_async/test.py +++ b/tests/integration/test_insert_into_distributed_sync_async/test.py @@ -23,7 +23,7 @@ def started_cluster(): for node in (node1, node2): node.query( """ -CREATE TABLE local_table(date Date, val UInt64) ENGINE = MergeTree(date, (date, val), 8192); +CREATE TABLE local_table(date Date, val UInt64) ENGINE = MergeTree() PARTITION BY toYYYYMM(date) ORDER BY (date, val); """ ) diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml index de0c930b8ab..295d1c8d3cc 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml @@ -3,6 +3,7 @@ 1 3 + 1 diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py index 7c2ce9f05f2..a5f92002450 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py @@ -44,7 +44,8 @@ CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'defaul ) remote.query( - "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)" + "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)", + settings={"allow_deprecated_syntax_for_merge_tree": 1}, ) instance_test_inserts_batching.query( """ @@ -65,7 +66,8 @@ CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', "CREATE MATERIALIZED VIEW local_view to distributed_on_local AS SELECT d,x FROM local_source" ) instance_test_inserts_local_cluster.query( - "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)" + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)", + settings={"allow_deprecated_syntax_for_merge_tree": 1}, ) instance_test_inserts_local_cluster.query( """ diff --git a/tests/integration/test_keeper_multinode_blocade_leader/test.py b/tests/integration/test_keeper_multinode_blocade_leader/test.py index c2d4039e122..d6d01a5d0a6 100644 --- a/tests/integration/test_keeper_multinode_blocade_leader/test.py +++ b/tests/integration/test_keeper_multinode_blocade_leader/test.py @@ -95,7 +95,10 @@ def test_blocade_leader(started_cluster): wait_nodes() try: for i, node in enumerate([node1, node2, node3]): - node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary") + node.query( + "CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query( "CREATE TABLE IF NOT EXISTS ordinary.t1 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t1', '{}') ORDER BY tuple()".format( i + 1 @@ -296,7 +299,10 @@ def test_blocade_leader_twice(started_cluster): wait_nodes() try: for i, node in enumerate([node1, node2, node3]): - node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary") + node.query( + "CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query( "CREATE TABLE IF NOT EXISTS ordinary.t2 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t2', '{}') ORDER BY tuple()".format( i + 1 diff --git a/tests/integration/test_materialized_mysql_database/configs/users.xml b/tests/integration/test_materialized_mysql_database/configs/users.xml index 4b7f5a1b109..0e116f115fe 100644 --- a/tests/integration/test_materialized_mysql_database/configs/users.xml +++ b/tests/integration/test_materialized_mysql_database/configs/users.xml @@ -3,7 +3,6 @@ 1 - Atomic 1 0 diff --git a/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml b/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml index f590ebff6b4..a00b6ca6b9a 100644 --- a/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml +++ b/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml @@ -3,7 +3,6 @@ 1 - Atomic 1 0 diff --git a/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml b/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml index e0fa0a9097b..3a7cc2537e5 100644 --- a/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml +++ b/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml @@ -3,7 +3,6 @@ 1 - Atomic 0 1 diff --git a/tests/integration/test_merge_tree_empty_parts/test.py b/tests/integration/test_merge_tree_empty_parts/test.py index 7ca275e96de..57bf49e6803 100644 --- a/tests/integration/test_merge_tree_empty_parts/test.py +++ b/tests/integration/test_merge_tree_empty_parts/test.py @@ -25,7 +25,7 @@ def started_cluster(): def test_empty_parts_alter_delete(started_cluster): node1.query( "CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1', d, key, 8192)" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1') PARTITION BY toYYYYMM(d) ORDER BY key" ) node1.query("INSERT INTO empty_parts_delete VALUES (toDate('2020-10-10'), 1, 'a')") @@ -44,7 +44,7 @@ def test_empty_parts_alter_delete(started_cluster): def test_empty_parts_summing(started_cluster): node1.query( "CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1', d, key, 8192)" + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1') PARTITION BY toYYYYMM(d) ORDER BY key" ) node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, 1)") diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py index e6ca4a78c25..f4acc4ac91e 100644 --- a/tests/integration/test_merge_tree_s3_restore/test.py +++ b/tests/integration/test_merge_tree_s3_restore/test.py @@ -93,7 +93,8 @@ def create_table( node.query( "CREATE DATABASE IF NOT EXISTS s3 ENGINE = {engine}".format( engine="Atomic" if db_atomic else "Ordinary" - ) + ), + settings={"allow_deprecated_database_ordinary": 1}, ) create_table_statement = """ diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py index d1843017b9f..7831cde7dea 100644 --- a/tests/integration/test_mutations_with_merge_tree/test.py +++ b/tests/integration/test_mutations_with_merge_tree/test.py @@ -17,7 +17,7 @@ def started_cluster(): try: cluster.start() instance_test_mutations.query( - """CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)""" + """CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY (a, date)""" ) instance_test_mutations.query( """INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1) SETTINGS force_index_by_date = 0, force_primary_key = 0""" diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index b396b58df10..f3df66631a5 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -14,7 +14,8 @@ def started_cluster(): try: cluster.start() q( - "CREATE DATABASE test ENGINE = Ordinary" + "CREATE DATABASE test ENGINE = Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different path in shadow/ with Atomic yield cluster diff --git a/tests/integration/test_parts_removal/__init__.py b/tests/integration/test_parts_removal/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_parts_removal/test.py b/tests/integration/test_parts_removal/test.py new file mode 100644 index 00000000000..4772178d63b --- /dev/null +++ b/tests/integration/test_parts_removal/test.py @@ -0,0 +1,71 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance("node", stay_alive=True) + + +def query_split(node, query): + return list( + map(lambda x: x.strip().split("\t"), node.query(query).strip().split("\n")) + ) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_parts_removal_on_abnormal_exit(start_cluster): + node.query( + """ + create table test_parts_removal (key Int) engine=MergeTree order by key; + insert into test_parts_removal values (1); -- all_1_1_0 + insert into test_parts_removal values (2); -- all_1_2_0 + optimize table test_parts_removal; -- all_2_2_0 + """ + ) + + parts = query_split( + node, "select name, _state from system.parts where table = 'test_parts_removal'" + ) + assert parts == [ + ["all_1_1_0", "Outdated"], + ["all_1_2_1", "Active"], + ["all_2_2_0", "Outdated"], + ] + + node.restart_clickhouse(kill=True) + + parts = query_split( + node, "select name, _state from system.parts where table = 'test_parts_removal'" + ) + assert parts == [ + ["all_1_1_0", "Outdated"], + ["all_1_2_1", "Active"], + ["all_2_2_0", "Outdated"], + ] + + node.query( + """ + detach table test_parts_removal; + attach table test_parts_removal; + """ + ) + + parts = query_split( + node, "select name, _state from system.parts where table = 'test_parts_removal'" + ) + assert parts == [ + ["all_1_2_1", "Active"], + ] diff --git a/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml b/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml index 7f8036c4f87..c654cf0791a 100644 --- a/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml +++ b/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml @@ -2,6 +2,7 @@ 0 + 1 diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index edd65ec002c..32b5e531fa8 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -66,7 +66,8 @@ def create_tables_old_format(name, nodes, shard): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}', date, id, 64) """.format( name=name, shard=shard, repl=i - ) + ), + settings={"allow_deprecated_syntax_for_merge_tree": 1}, ) @@ -495,7 +496,7 @@ def test_polymorphic_parts_diff_versions_2(start_cluster_diff_versions): with pytest.raises(Exception): node_old.query("SYSTEM SYNC REPLICA polymorphic_table_2", timeout=3) - node_old.restart_with_latest_version() + node_old.restart_with_latest_version(fix_metadata=True) node_old.query("SYSTEM SYNC REPLICA polymorphic_table_2", timeout=20) @@ -720,7 +721,8 @@ def test_in_memory_alters(start_cluster): def test_polymorphic_parts_index(start_cluster): node1.query( - "CREATE DATABASE test_index ENGINE=Ordinary" + "CREATE DATABASE test_index ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different paths with Atomic node1.query( """ diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index dd5b3a09ca5..aabf3507d8f 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -320,6 +320,20 @@ def test_predefined_connection_configuration(started_cluster): cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") +def test_postgres_database_old_syntax(started_cluster): + conn = get_postgres_conn(started_cluster, True) + cursor = conn.cursor() + + node1.query( + """ + DROP DATABASE IF EXISTS test_database; + CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 1); + """ + ) + create_postgres_table(cursor, "test_table") + assert "test_table" in node1.query("SHOW TABLES FROM test_database") + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_random_inserts/test.py b/tests/integration/test_random_inserts/test.py index 4d6aaa9276d..9ac0c5b024c 100644 --- a/tests/integration/test_random_inserts/test.py +++ b/tests/integration/test_random_inserts/test.py @@ -44,7 +44,7 @@ def test_random_inserts(started_cluster): node1.query( """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, i UInt32, s String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, i, 8192)""" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY i""" ) with PartitionManager() as pm_random_drops: diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index 7ce79d9aca8..579b22286b9 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -20,13 +20,13 @@ def _fill_nodes(nodes, shard): CREATE DATABASE test; CREATE TABLE real_table(date Date, id UInt32, dummy UInt32) - ENGINE = MergeTree(date, id, 8192); + ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY id; CREATE TABLE other_table(date Date, id UInt32, dummy UInt32) - ENGINE = MergeTree(date, id, 8192); + ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY id; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 92c109974e1..11ca0d2f962 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -96,13 +96,16 @@ def test_create_replicated_table(started_cluster): "Explicit zookeeper_path and replica_name are specified" in main_node.query_and_get_error( "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " - "ENGINE=ReplicatedMergeTree('/test/tmp', 'r', d, k, 8192);" + "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" ) ) - assert "Old syntax is not allowed" in main_node.query_and_get_error( - "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " - "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" + assert ( + "This syntax for *MergeTree engine is deprecated" + in main_node.query_and_get_error( + "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" + ) ) main_node.query( @@ -393,7 +396,7 @@ def test_alters_from_different_replicas(started_cluster): main_node.query( "CREATE TABLE testdb.concurrent_test " "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " - "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);" + "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" ) main_node.query( @@ -443,7 +446,7 @@ def test_alters_from_different_replicas(started_cluster): " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" " `AddedNested2.B` Array(UInt64)\\n)\\n" - "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192)" + "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" ) assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index 00367daad33..eb2b14ffb1a 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -52,7 +52,10 @@ def test_replicated_merge_tree_defaults_compatibility(started_cluster): """ for node in (node1, node2): - node.query("CREATE DATABASE test ENGINE = Ordinary") + node.query( + "CREATE DATABASE test ENGINE = Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) node.query(create_query.format(replica=node.name)) node1.query("DETACH TABLE test.table") diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/test.py b/tests/integration/test_replicated_merge_tree_s3_restore/test.py index 904bcfa4280..d743dedbdde 100644 --- a/tests/integration/test_replicated_merge_tree_s3_restore/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_restore/test.py @@ -76,7 +76,8 @@ def create_table(node, table_name, schema, attach=False, db_atomic=False, uuid=" "CREATE DATABASE IF NOT EXISTS s3 {on_cluster} ENGINE = {engine}".format( engine="Atomic" if db_atomic else "Ordinary", on_cluster="ON CLUSTER '{cluster}'", - ) + ), + settings={"allow_deprecated_database_ordinary": 1}, ) create_table_statement = """ diff --git a/tests/integration/test_replication_credentials/test.py b/tests/integration/test_replication_credentials/test.py index e5313cb6bd4..79588fcd38b 100644 --- a/tests/integration/test_replication_credentials/test.py +++ b/tests/integration/test_replication_credentials/test.py @@ -10,7 +10,7 @@ def _fill_nodes(nodes, shard): """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_send_request_to_leader_replica/test.py b/tests/integration/test_send_request_to_leader_replica/test.py index 60df18bf7d3..b56e1315672 100644 --- a/tests/integration/test_send_request_to_leader_replica/test.py +++ b/tests/integration/test_send_request_to_leader_replica/test.py @@ -40,7 +40,7 @@ def started_cluster(): node.query( """ CREATE TABLE sometable(date Date, id UInt32, value Int32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ), @@ -51,7 +51,7 @@ def started_cluster(): node.query( """ CREATE TABLE someothertable(date Date, id UInt32, value Int32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ), diff --git a/tests/integration/test_server_initialization/clickhouse_path/metadata/default.sql b/tests/integration/test_server_initialization/clickhouse_path/metadata/default.sql new file mode 100644 index 00000000000..9fae1d57726 --- /dev/null +++ b/tests/integration/test_server_initialization/clickhouse_path/metadata/default.sql @@ -0,0 +1 @@ +ATTACH DATABASE default ENGINE=Ordinary \ No newline at end of file diff --git a/tests/integration/test_server_start_and_ip_conversions/__init__.py b/tests/integration/test_server_start_and_ip_conversions/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_server_start_and_ip_conversions/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_server_start_and_ip_conversions/test.py b/tests/integration/test_server_start_and_ip_conversions/test.py new file mode 100644 index 00000000000..abb6a546f64 --- /dev/null +++ b/tests/integration/test_server_start_and_ip_conversions/test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import logging +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", stay_alive=True) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_restart_success_ipv4(): + node.query( + """ + CREATE TABLE ipv4_test + ( + id UInt64, + value String + ) ENGINE=MergeTree ORDER BY id""", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) + + node.query( + "ALTER TABLE ipv4_test MODIFY COLUMN value IPv4 DEFAULT ''", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) + + node.restart_clickhouse() + + assert node.query("SELECT 1") == "1\n" + + +def test_restart_success_ipv6(): + node.query( + """ + CREATE TABLE ipv6_test + ( + id UInt64, + value String + ) ENGINE=MergeTree ORDER BY id""", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) + + node.query( + "ALTER TABLE ipv6_test MODIFY COLUMN value IPv6 DEFAULT ''", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) + + node.restart_clickhouse() + + assert node.query("SELECT 1") == "1\n" diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index fb00403b952..5ac8b4670f9 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -113,7 +113,7 @@ def test_read_table_expired(started_cluster): ) assert False, "Exception have to be thrown" except Exception as ex: - assert "DB::Exception: kinit failure:" in str(ex) + assert "DB::Exception: KerberosInit failure:" in str(ex) started_cluster.unpause_container("hdfskerberos") diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 6347ba89c16..7856361deda 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -128,6 +128,48 @@ def test_kafka_json_as_string(kafka_cluster): ) +def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster): + # Ticket should be expired after the wait time + # On run of SELECT query new ticket should be requested and SELECT query should run fine. + + kafka_produce( + kafka_cluster, + "kafka_json_as_string", + [ + '{"t": 123, "e": {"x": "woof"} }', + "", + '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}', + ], + ) + + instance.query( + """ + CREATE TABLE test.kafka (field String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', + kafka_topic_list = 'kafka_json_as_string', + kafka_commit_on_select = 1, + kafka_group_name = 'kafka_json_as_string', + kafka_format = 'JSONAsString', + kafka_flush_interval_ms=1000; + """ + ) + + time.sleep(45) # wait for ticket expiration + + result = instance.query("SELECT * FROM test.kafka;") + expected = """\ +{"t": 123, "e": {"x": "woof"} } +{"t": 124, "e": {"x": "test"} } +{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} +""" + assert TSV(result) == TSV(expected) + assert instance.contains_in_log( + "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" + ) + + def test_kafka_json_as_string_no_kdc(kafka_cluster): # When the test is run alone (not preceded by any other kerberized kafka test), # we need a ticket to @@ -182,7 +224,7 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") assert instance.contains_in_log("Ticket expired") - assert instance.contains_in_log("Kerberos ticket refresh failed") + assert instance.contains_in_log("KerberosInit failure:") if __name__ == "__main__": diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 8366ca5dc25..1fc0475419c 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -433,6 +433,7 @@ def test_datetime_with_timezone(started_cluster): def test_postgres_ndim(started_cluster): cursor = started_cluster.postgres_conn.cursor() + cursor.execute("DROP TABLE IF EXISTS arr1, arr2") cursor.execute("CREATE TABLE arr1 (a Integer[])") @@ -452,6 +453,20 @@ def test_postgres_ndim(started_cluster): assert result.strip() == "Array(Array(Nullable(Int32)))" cursor.execute("DROP TABLE arr1, arr2") + cursor.execute("DROP SCHEMA IF EXISTS ndim_schema CASCADE") + cursor.execute("CREATE SCHEMA ndim_schema") + cursor.execute("CREATE TABLE ndim_schema.arr1 (a integer[])") + cursor.execute("INSERT INTO ndim_schema.arr1 SELECT '{{1}, {2}}'") + # The point is in creating a table via 'as select *', in postgres att_ndim will not be correct in this case. + cursor.execute("CREATE TABLE ndim_schema.arr2 AS SELECT * FROM ndim_schema.arr1") + result = node1.query( + """SELECT toTypeName(a) FROM postgresql(postgres1, schema='ndim_schema', table='arr2')""" + ) + assert result.strip() == "Array(Array(Nullable(Int32)))" + + cursor.execute("DROP TABLE ndim_schema.arr1, ndim_schema.arr2") + cursor.execute("DROP SCHEMA ndim_schema CASCADE") + def test_postgres_on_conflict(started_cluster): cursor = started_cluster.postgres_conn.cursor() diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index f497a5bb2c0..4da55d09b85 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2784,9 +2784,3 @@ def test_rabbitmq_address(rabbitmq_cluster): break time.sleep(1) assert result.strip() == "kek" - - -if __name__ == "__main__": - cluster.start() - input("Cluster created, press any key to destroy...") - cluster.shutdown() diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 775706f4df6..b908ba0eb33 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -26,9 +26,13 @@ def started_cluster(): try: cluster.start() node1.query( - "CREATE DATABASE test ENGINE=Ordinary" + "CREATE DATABASE test ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, ) # Different paths with Atomic - node2.query("CREATE DATABASE test ENGINE=Ordinary") + node2.query( + "CREATE DATABASE test ENGINE=Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) yield cluster finally: diff --git a/tests/integration/test_transactions/test.py b/tests/integration/test_transactions/test.py index 8983e70b4cb..2dfdc889856 100644 --- a/tests/integration/test_transactions/test.py +++ b/tests/integration/test_transactions/test.py @@ -108,13 +108,13 @@ def test_rollback_unfinished_on_restart(start_cluster): assert ( res == "0_2_2_0\t1\ttid0\tcsn1_\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" - "0_2_4_1\t0\ttid4\tcsn18446744073709551615_\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" + "0_2_4_1\t0\ttid4\tcsn18446744073709551615_\ttid0\tcsn0_\n" "0_4_4_0\t1\ttid2\tcsn_2\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" - "0_8_8_0\t0\ttid5\tcsn18446744073709551615_\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" + "0_8_8_0\t0\ttid5\tcsn18446744073709551615_\ttid0\tcsn0_\n" "1_1_1_0\t0\ttid0\tcsn1_\ttid1\tcsn_1\n" "1_3_3_0\t1\ttid2\tcsn_2\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" - "1_3_3_0_7\t0\ttid3\tcsn18446744073709551615_\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" + "1_3_3_0_7\t0\ttid3\tcsn18446744073709551615_\ttid0\tcsn0_\n" "1_5_5_0\t1\ttid6\tcsn_6\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" - "1_6_6_0\t0\ttid3\tcsn18446744073709551615_\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" - "1_6_6_0_7\t0\ttid3\tcsn18446744073709551615_\t(0,0,'00000000-0000-0000-0000-000000000000')\tcsn0_\n" + "1_6_6_0\t0\ttid3\tcsn18446744073709551615_\ttid0\tcsn0_\n" + "1_6_6_0_7\t0\ttid3\tcsn18446744073709551615_\ttid0\tcsn0_\n" ) diff --git a/tests/integration/test_union_header/test.py b/tests/integration/test_union_header/test.py index f883057c1d8..2e7f6cb399a 100644 --- a/tests/integration/test_union_header/test.py +++ b/tests/integration/test_union_header/test.py @@ -27,7 +27,7 @@ def started_cluster(): log_type UInt32, account_id String ) - ENGINE = MergeTree(event_date, (event_time, account_id), 8192); + ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_time, account_id); """ ) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 2971cbc9792..a21186bba8d 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -58,8 +58,8 @@ def test_mutate_and_upgrade(start_cluster): node2.query("DETACH TABLE mt") # stop being leader node1.query("DETACH TABLE mt") # stop being leader - node1.restart_with_latest_version(signal=9) - node2.restart_with_latest_version(signal=9) + node1.restart_with_latest_version(signal=9, fix_metadata=True) + node2.restart_with_latest_version(signal=9, fix_metadata=True) # After hard restart table can be in readonly mode exec_query_with_retry( @@ -111,7 +111,7 @@ def test_upgrade_while_mutation(start_cluster): node3.query("ALTER TABLE mt1 DELETE WHERE id % 2 == 0") node3.query("DETACH TABLE mt1") # stop being leader - node3.restart_with_latest_version(signal=9) + node3.restart_with_latest_version(signal=9, fix_metadata=True) # checks for readonly exec_query_with_retry(node3, "OPTIMIZE TABLE mt1", sleep_time=5, retry_count=60) diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index d3d90ca0d4f..65f82c2286b 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -48,7 +48,7 @@ def test_chroot_with_same_root(started_cluster): node.query( """ CREATE TABLE simple (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ) @@ -68,7 +68,7 @@ def test_chroot_with_different_root(started_cluster): node.query( """ CREATE TABLE simple_different (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ) diff --git a/tests/integration/test_zookeeper_config/test_password.py b/tests/integration/test_zookeeper_config/test_password.py index 580b426db6f..71f059b3277 100644 --- a/tests/integration/test_zookeeper_config/test_password.py +++ b/tests/integration/test_zookeeper_config/test_password.py @@ -35,7 +35,7 @@ def test_identity(started_cluster): node1.query( """ CREATE TABLE simple (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node1.name ) @@ -45,6 +45,6 @@ def test_identity(started_cluster): node2.query( """ CREATE TABLE simple (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1') PARTITION BY toYYYYMM(date) ORDER BY id; """ ) diff --git a/tests/performance/json_type.xml b/tests/performance/json_type.xml new file mode 100644 index 00000000000..b6406f52579 --- /dev/null +++ b/tests/performance/json_type.xml @@ -0,0 +1,41 @@ + + + 1 + + + + + + + json1 + + '{"k1":1, "k2": "some"}' + + + + json2 + + '{"col' || toString(number % 100) || '":' || toString(number) || '}' + + + + json3 + + '{"k1":[{"k2":"aaa","k3":[{"k4":"bbb"},{"k4":"ccc"}]},{"k2":"ddd","k3":[{"k4":"eee"},{"k4":"fff"}]}]}' + + + + + CREATE TABLE t_json_1(data JSON) ENGINE = MergeTree ORDER BY tuple() + CREATE TABLE t_json_2(data JSON) ENGINE = MergeTree ORDER BY tuple() + CREATE TABLE t_json_3(data JSON) ENGINE = MergeTree ORDER BY tuple() + + INSERT INTO t_json_1 SELECT materialize({json1}) FROM numbers(200000) + INSERT INTO t_json_2 SELECT {json2} FROM numbers(100000) + INSERT INTO t_json_3 SELECT materialize({json3}) FROM numbers_mt(100000) + + DROP TABLE IF EXISTS t_json_1 + DROP TABLE IF EXISTS t_json_2 + DROP TABLE IF EXISTS t_json_3 + diff --git a/tests/performance/merge_table_streams.xml b/tests/performance/merge_table_streams.xml index efeb4547f37..1e053c98738 100644 --- a/tests/performance/merge_table_streams.xml +++ b/tests/performance/merge_table_streams.xml @@ -1,6 +1,7 @@ 5 + 1